From 2e663606e9d4f62a4e8cd7ec4360532a97b1489d Mon Sep 17 00:00:00 2001 From: Vijay-Nirmal Date: Wed, 5 Feb 2025 13:38:50 +0530 Subject: [PATCH 1/9] Added support to expire specfic member of a sorted set by adding family of commands --- hexpire.diff | 849 ++++++++++++++++++ libs/host/Configuration/Options.cs | 5 + libs/host/defaults.conf | 3 + libs/resources/RespCommandsDocs.json | 507 +++++++++++ libs/resources/RespCommandsInfo.json | 250 ++++++ libs/server/API/GarnetApiObjectCommands.cs | 16 + libs/server/API/GarnetWatchApi.cs | 7 + libs/server/API/IGarnetApi.cs | 39 + .../Objects/SortedSet/SortedSetObject.cs | 302 ++++++- .../Objects/SortedSet/SortedSetObjectImpl.cs | 212 ++++- libs/server/Objects/Types/GarnetObject.cs | 2 + libs/server/Resp/AdminCommands.cs | 31 + libs/server/Resp/BasicCommands.cs | 1 - libs/server/Resp/CmdStrings.cs | 3 + libs/server/Resp/Objects/SortedSetCommands.cs | 260 ++++++ libs/server/Resp/Parser/RespCommand.cs | 50 +- libs/server/Resp/RespServerSession.cs | 9 + libs/server/Servers/GarnetServerOptions.cs | 5 + .../Session/ObjectStore/SortedSetOps.cs | 123 +++ libs/server/StoreWrapper.cs | 48 + .../CommandInfoUpdater/SupportedCommand.cs | 10 + .../RedirectTests/BaseCommand.cs | 178 ++++ .../ClusterSlotVerificationTests.cs | 70 ++ test/Garnet.test/Resp/ACL/RespCommandTests.cs | 159 ++++ website/docs/commands/data-structures.md | 235 +++++ website/docs/commands/garnet-specific.md | 19 + website/docs/getting-started/configuration.md | 1 + 27 files changed, 3369 insertions(+), 25 deletions(-) create mode 100644 hexpire.diff diff --git a/hexpire.diff b/hexpire.diff new file mode 100644 index 0000000000..e4d96fcbea --- /dev/null +++ b/hexpire.diff @@ -0,0 +1,849 @@ +diff --git a/libs/server/Objects/Hash/HashObject.cs b/libs/server/Objects/Hash/HashObject.cs +index bfa3a8b4..8ba8c21b 100644 +--- a/libs/server/Objects/Hash/HashObject.cs ++++ b/libs/server/Objects/Hash/HashObject.cs +@@ -5,6 +5,8 @@ using System; + using System.Collections.Generic; + using System.Diagnostics; + using System.IO; ++using System.Linq; ++using System.Runtime.CompilerServices; + using Garnet.common; + using Tsavorite.core; + +@@ -17,6 +19,10 @@ namespace Garnet.server + /// + public enum HashOperation : byte + { ++ HCOLLECT, ++ HEXPIRE, ++ HTTL, ++ HPERSIST, + HGET, + HMGET, + HSET, +@@ -42,6 +48,11 @@ namespace Garnet.server + public unsafe partial class HashObject : GarnetObjectBase + { + readonly Dictionary hash; ++ Dictionary expirationTimes; ++ PriorityQueue expirationQueue; ++ ++ // Byte #31 is used to denote if key has expiration (1) or not (0) ++ private const int ExpirationBitMask = 1 << 31; + + /// + /// Constructor +@@ -63,9 +74,29 @@ namespace Garnet.server + int count = reader.ReadInt32(); + for (int i = 0; i < count; i++) + { +- var item = reader.ReadBytes(reader.ReadInt32()); ++ var keyLength = reader.ReadInt32(); ++ var hasExpiration = (keyLength & ExpirationBitMask) != 0; ++ keyLength &= ~ExpirationBitMask; ++ var item = reader.ReadBytes(keyLength); + var value = reader.ReadBytes(reader.ReadInt32()); +- hash.Add(item, value); ++ ++ if (hasExpiration) ++ { ++ var expiration = reader.ReadInt64(); ++ var isExpired = expiration < DateTimeOffset.UtcNow.Ticks; ++ if (!isExpired) ++ { ++ hash.Add(item, value); ++ InitializeExpirationStructures(); ++ expirationTimes.Add(item, expiration); ++ expirationQueue.Enqueue(item, expiration); ++ UpdateExpirationSize(item, true); ++ } ++ } ++ else ++ { ++ hash.Add(item, value); ++ } + + this.UpdateSize(item, value); + } +@@ -74,10 +105,12 @@ namespace Garnet.server + /// + /// Copy constructor + /// +- public HashObject(Dictionary hash, long expiration, long size) ++ public HashObject(Dictionary hash, Dictionary expirationTimes, PriorityQueue expirationQueue, long expiration, long size) + : base(expiration, size) + { + this.hash = hash; ++ this.expirationTimes = expirationTimes; ++ this.expirationQueue = expirationQueue; + } + + /// +@@ -88,16 +121,30 @@ namespace Garnet.server + { + base.DoSerialize(writer); + +- int count = hash.Count; ++ DeleteExpiredItems(); ++ ++ int count = hash.Count; // Since expired items are already deleted, no need to worry about expiring items + writer.Write(count); + foreach (var kvp in hash) + { ++ if (expirationTimes is not null && expirationTimes.TryGetValue(kvp.Key, out var expiration)) ++ { ++ writer.Write(kvp.Key.Length | ExpirationBitMask); ++ writer.Write(kvp.Key); ++ writer.Write(kvp.Value.Length); ++ writer.Write(kvp.Value); ++ writer.Write(expiration); ++ count--; ++ continue; ++ } ++ + writer.Write(kvp.Key.Length); + writer.Write(kvp.Key); + writer.Write(kvp.Value.Length); + writer.Write(kvp.Value); + count--; + } ++ + Debug.Assert(count == 0); + } + +@@ -105,7 +152,7 @@ namespace Garnet.server + public override void Dispose() { } + + /// +- public override GarnetObjectBase Clone() => new HashObject(hash, Expiration, Size); ++ public override GarnetObjectBase Clone() => new HashObject(hash, expirationTimes, expirationQueue, Expiration, Size); + + /// + public override unsafe bool Operate(ref ObjectInput input, ref SpanByteAndMemory output, out long sizeChange, out bool removeKey) +@@ -152,6 +199,15 @@ namespace Garnet.server + case HashOperation.HEXISTS: + HashExists(ref input, _output); + break; ++ case HashOperation.HEXPIRE: ++ HashExpire(ref input, ref output); ++ break; ++ case HashOperation.HTTL: ++ HashTimeToLive(ref input, ref output); ++ break; ++ case HashOperation.HPERSIST: ++ HashPersist(ref input, ref output); ++ break; + case HashOperation.HKEYS: + HashGetKeysOrValues(ref input, ref output); + break; +@@ -170,6 +226,9 @@ namespace Garnet.server + case HashOperation.HRANDFIELD: + HashRandomField(ref input, ref output); + break; ++ case HashOperation.HCOLLECT: ++ HashCollect(ref input, _output); ++ break; + case HashOperation.HSCAN: + if (ObjectUtils.ReadScanInput(ref input, ref output, out var cursorInput, out var pattern, + out var patternLength, out var limitCount, out bool isNoValue, out var error)) +@@ -202,6 +261,38 @@ namespace Garnet.server + Debug.Assert(this.Size >= MemoryUtils.DictionaryOverhead); + } + ++ [MethodImpl(MethodImplOptions.AggressiveInlining)] ++ private void InitializeExpirationStructures() ++ { ++ if (expirationTimes is null) ++ { ++ expirationTimes = new Dictionary(ByteArrayComparer.Instance); ++ expirationQueue = new PriorityQueue(); ++ this.Size += MemoryUtils.DictionaryOverhead + MemoryUtils.PriorityQueueOverhead; ++ } ++ } ++ ++ [MethodImpl(MethodImplOptions.AggressiveInlining)] ++ private void UpdateExpirationSize(ReadOnlySpan key, bool add = true) ++ { ++ // Account for dictionary entry and priority queue entry ++ var size = IntPtr.Size + sizeof(long) + MemoryUtils.DictionaryEntryOverhead ++ + IntPtr.Size + sizeof(long) + MemoryUtils.PriorityQueueEntryOverhead; ++ this.Size += add ? size : -size; ++ } ++ ++ [MethodImpl(MethodImplOptions.AggressiveInlining)] ++ private void CleanupExpirationStructures() ++ { ++ if (expirationTimes.Count == 0) ++ { ++ this.Size -= (IntPtr.Size + sizeof(long) + MemoryUtils.PriorityQueueOverhead) * expirationQueue.Count; ++ this.Size -= MemoryUtils.DictionaryOverhead + MemoryUtils.PriorityQueueOverhead; ++ expirationTimes = null; ++ expirationQueue = null; ++ } ++ } ++ + /// + public override unsafe void Scan(long start, out List items, out long cursor, int count = 10, byte* pattern = default, int patternLength = 0, bool isNoValue = false) + { +@@ -217,8 +308,15 @@ namespace Garnet.server + // Hashset has key and value, so count is multiplied by 2 + count = isNoValue ? count : count * 2; + int index = 0; ++ var expiredKeysCount = 0; + foreach (var item in hash) + { ++ if (IsExpired(item.Key)) ++ { ++ expiredKeysCount++; ++ continue; ++ } ++ + if (index < start) + { + index++; +@@ -256,8 +354,241 @@ namespace Garnet.server + } + + // Indicates end of collection has been reached. +- if (cursor == hash.Count) ++ if (cursor + expiredKeysCount == hash.Count) + cursor = 0; + } ++ ++ [MethodImpl(MethodImplOptions.AggressiveInlining)] ++ private bool IsExpired(byte[] key) => expirationTimes is not null && expirationTimes.TryGetValue(key, out var expiration) && expiration < DateTimeOffset.UtcNow.Ticks; ++ ++ private void DeleteExpiredItems() ++ { ++ if (expirationTimes is null) ++ return; ++ ++ while (expirationQueue.TryPeek(out var key, out var expiration) && expiration < DateTimeOffset.UtcNow.Ticks) ++ { ++ // expirationTimes and expirationQueue will be out of sync when user is updating the expire time of key which already has some TTL. ++ // PriorityQueue Doesn't have update option, so we will just enqueue the new expiration and already treat expirationTimes as the source of truth ++ if (expirationTimes.TryGetValue(key, out var actualExpiration) && actualExpiration == expiration) ++ { ++ expirationTimes.Remove(key); ++ expirationQueue.Dequeue(); ++ UpdateExpirationSize(key, false); ++ if (hash.TryGetValue(key, out var value)) ++ { ++ hash.Remove(key); ++ UpdateSize(key, value, false); ++ } ++ } ++ else ++ { ++ expirationQueue.Dequeue(); ++ this.Size -= MemoryUtils.PriorityQueueEntryOverhead + IntPtr.Size + sizeof(long); ++ } ++ } ++ ++ CleanupExpirationStructures(); ++ } ++ ++ private bool TryGetValue(byte[] key, out byte[] value) ++ { ++ value = default; ++ if (IsExpired(key)) ++ { ++ return false; ++ } ++ return hash.TryGetValue(key, out value); ++ } ++ ++ private bool Remove(byte[] key, out byte[] value) ++ { ++ DeleteExpiredItems(); ++ var result = hash.Remove(key, out value); ++ if (result) ++ { ++ UpdateSize(key, value, false); ++ } ++ return result; ++ } ++ ++ private int Count() ++ { ++ if (expirationTimes is null) ++ { ++ return hash.Count; ++ } ++ ++ var expiredKeysCount = 0; ++ foreach (var item in expirationTimes) ++ { ++ if (IsExpired(item.Key)) ++ { ++ expiredKeysCount++; ++ } ++ } ++ ++ return hash.Count - expiredKeysCount; ++ } ++ ++ [MethodImpl(MethodImplOptions.AggressiveInlining)] ++ private bool HasExpirableItems() ++ { ++ return expirationTimes is not null; ++ } ++ ++ private bool ContainsKey(byte[] key) ++ { ++ var result = hash.ContainsKey(key); ++ if (result && IsExpired(key)) ++ { ++ return false; ++ } ++ ++ return result; ++ } ++ ++ [MethodImpl(MethodImplOptions.AggressiveInlining)] ++ private void Add(byte[] key, byte[] value) ++ { ++ DeleteExpiredItems(); ++ hash.Add(key, value); ++ UpdateSize(key, value); ++ } ++ ++ private void Set(byte[] key, byte[] value) ++ { ++ DeleteExpiredItems(); ++ hash[key] = value; ++ // Skip overhead as existing item is getting replaced. ++ this.Size += Utility.RoundUp(value.Length, IntPtr.Size) - ++ Utility.RoundUp(value.Length, IntPtr.Size); ++ ++ // To persist the key, if it has an expiration ++ if (expirationTimes is not null && expirationTimes.TryGetValue(key, out var currentExpiration)) ++ { ++ expirationTimes.Remove(key); ++ this.Size -= IntPtr.Size + sizeof(long) + MemoryUtils.DictionaryEntryOverhead; ++ CleanupExpirationStructures(); ++ } ++ } ++ ++ private void SetWithoutPersist(byte[] key, byte[] value) ++ { ++ DeleteExpiredItems(); ++ hash[key] = value; ++ // Skip overhead as existing item is getting replaced. ++ this.Size += Utility.RoundUp(value.Length, IntPtr.Size) - ++ Utility.RoundUp(value.Length, IntPtr.Size); ++ } ++ ++ private int SetExpiration(byte[] key, long expiration, ExpireOption expireOption) ++ { ++ if (!ContainsKey(key)) ++ { ++ return (int)ExpireResult.KeyNotFound; ++ } ++ ++ if (expiration <= DateTimeOffset.UtcNow.Ticks) ++ { ++ Remove(key, out _); ++ return (int)ExpireResult.KeyAlreadyExpired; ++ } ++ ++ InitializeExpirationStructures(); ++ ++ if (expirationTimes.TryGetValue(key, out var currentExpiration)) ++ { ++ if (expireOption.HasFlag(ExpireOption.NX) || ++ (expireOption.HasFlag(ExpireOption.GT) && expiration <= currentExpiration) || ++ (expireOption.HasFlag(ExpireOption.LT) && expiration >= currentExpiration)) ++ { ++ return (int)ExpireResult.ExpireConditionNotMet; ++ } ++ ++ expirationTimes[key] = expiration; ++ expirationQueue.Enqueue(key, expiration); ++ // Size of dictionary entry already accounted for as the key already exists ++ this.Size += IntPtr.Size + sizeof(long) + MemoryUtils.PriorityQueueEntryOverhead; ++ } ++ else ++ { ++ if (expireOption.HasFlag(ExpireOption.XX) || expireOption.HasFlag(ExpireOption.GT)) ++ { ++ return (int)ExpireResult.ExpireConditionNotMet; ++ } ++ ++ expirationTimes[key] = expiration; ++ expirationQueue.Enqueue(key, expiration); ++ UpdateExpirationSize(key); ++ } ++ ++ return (int)ExpireResult.ExpireUpdated; ++ } ++ ++ private int Persist(byte[] key) ++ { ++ if (!ContainsKey(key)) ++ { ++ return -2; ++ } ++ ++ if (expirationTimes is not null && expirationTimes.TryGetValue(key, out var currentExpiration)) ++ { ++ expirationTimes.Remove(key); ++ this.Size -= IntPtr.Size + sizeof(long) + MemoryUtils.DictionaryEntryOverhead; ++ CleanupExpirationStructures(); ++ return 1; ++ } ++ ++ return -1; ++ } ++ ++ private long GetExpiration(byte[] key) ++ { ++ if (!ContainsKey(key)) ++ { ++ return -2; ++ } ++ ++ if (expirationTimes.TryGetValue(key, out var expiration)) ++ { ++ return expiration; ++ } ++ ++ return -1; ++ } ++ ++ private KeyValuePair ElementAt(int index) ++ { ++ if (HasExpirableItems()) ++ { ++ var currIndex = 0; ++ foreach (var item in hash) ++ { ++ if (IsExpired(item.Key)) ++ { ++ continue; ++ } ++ ++ if (currIndex++ == index) ++ { ++ return item; ++ } ++ } ++ ++ throw new ArgumentOutOfRangeException("index is outside the bounds of the source sequence."); ++ } ++ ++ return hash.ElementAt(index); ++ } ++ } ++ ++ enum ExpireResult ++ { ++ KeyNotFound = -2, ++ ExpireConditionNotMet = 0, ++ ExpireUpdated = 1, ++ KeyAlreadyExpired = 2, + } + } +\ No newline at end of file +diff --git a/libs/server/Objects/Hash/HashObjectImpl.cs b/libs/server/Objects/Hash/HashObjectImpl.cs +index 674aebfd..14f6a84a 100644 +--- a/libs/server/Objects/Hash/HashObjectImpl.cs ++++ b/libs/server/Objects/Hash/HashObjectImpl.cs +@@ -33,7 +33,7 @@ namespace Garnet.server + { + var key = input.parseState.GetArgSliceByRef(0).SpanByte.ToByteArray(); + +- if (hash.TryGetValue(key, out var hashValue)) ++ if (TryGetValue(key, out var hashValue)) + { + while (!RespWriteUtils.WriteBulkString(hashValue, ref curr, end)) + ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); +@@ -75,7 +75,7 @@ namespace Garnet.server + { + var key = input.parseState.GetArgSliceByRef(i).SpanByte.ToByteArray(); + +- if (hash.TryGetValue(key, out var hashValue)) ++ if (TryGetValue(key, out var hashValue)) + { + while (!RespWriteUtils.WriteBulkString(hashValue, ref curr, end)) + ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); +@@ -115,17 +115,24 @@ namespace Garnet.server + { + if (respProtocolVersion < 3) + { +- while (!RespWriteUtils.WriteArrayLength(hash.Count * 2, ref curr, end)) ++ while (!RespWriteUtils.WriteArrayLength(Count() * 2, ref curr, end)) + ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); + } + else + { +- while (!RespWriteUtils.WriteMapLength(hash.Count, ref curr, end)) ++ while (!RespWriteUtils.WriteMapLength(Count(), ref curr, end)) + ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); + } + ++ var isExpirable = HasExpirableItems(); ++ + foreach (var item in hash) + { ++ if (isExpirable && IsExpired(item.Key)) ++ { ++ continue; ++ } ++ + while (!RespWriteUtils.WriteBulkString(item.Key, ref curr, end)) + ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); + while (!RespWriteUtils.WriteBulkString(item.Value, ref curr, end)) +@@ -151,17 +158,16 @@ namespace Garnet.server + { + var key = input.parseState.GetArgSliceByRef(i).SpanByte.ToByteArray(); + +- if (hash.Remove(key, out var hashValue)) ++ if (Remove(key, out var hashValue)) + { + _output->result1++; +- this.UpdateSize(key, hashValue, false); + } + } + } + + private void HashLength(byte* output) + { +- ((ObjectOutputHeader*)output)->result1 = hash.Count; ++ ((ObjectOutputHeader*)output)->result1 = Count(); + } + + private void HashStrLength(ref ObjectInput input, byte* output) +@@ -170,7 +176,7 @@ namespace Garnet.server + *_output = default; + + var key = input.parseState.GetArgSliceByRef(0).SpanByte.ToByteArray(); +- _output->result1 = hash.TryGetValue(key, out var hashValue) ? hashValue.Length : 0; ++ _output->result1 = TryGetValue(key, out var hashValue) ? hashValue.Length : 0; + } + + private void HashExists(ref ObjectInput input, byte* output) +@@ -179,7 +185,7 @@ namespace Garnet.server + *_output = default; + + var field = input.parseState.GetArgSliceByRef(0).SpanByte.ToByteArray(); +- _output->result1 = hash.ContainsKey(field) ? 1 : 0; ++ _output->result1 = ContainsKey(field) ? 1 : 0; + } + + private void HashRandomField(ref ObjectInput input, ref SpanByteAndMemory output) +@@ -204,11 +210,21 @@ namespace Garnet.server + { + if (includedCount) + { +- if (countParameter > 0 && countParameter > hash.Count) +- countParameter = hash.Count; ++ var count = Count(); ++ ++ if (count == 0) // This can happen because of expiration but RMW operation haven't applied yet ++ { ++ while (!RespWriteUtils.WriteEmptyArray(ref curr, end)) ++ ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); ++ _output.result1 = 0; ++ return; ++ } ++ ++ if (countParameter > 0 && countParameter > count) ++ countParameter = count; + + var absCount = Math.Abs(countParameter); +- var indexes = RandomUtils.PickKRandomIndexes(hash.Count, absCount, seed, countParameter > 0); ++ var indexes = RandomUtils.PickKRandomIndexes(count, absCount, seed, countParameter > 0); + + // Write the size of the array reply + while (!RespWriteUtils.WriteArrayLength(withValues ? absCount * 2 : absCount, ref curr, end)) +@@ -216,7 +232,7 @@ namespace Garnet.server + + foreach (var index in indexes) + { +- var pair = hash.ElementAt(index); ++ var pair = ElementAt(index); + while (!RespWriteUtils.WriteBulkString(pair.Key, ref curr, end)) + ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); + +@@ -232,8 +248,17 @@ namespace Garnet.server + else // No count parameter is present, we just return a random field + { + // Write a bulk string value of a random field from the hash value stored at key. +- var index = RandomUtils.PickRandomIndex(hash.Count, seed); +- var pair = hash.ElementAt(index); ++ var count = Count(); ++ if (count == 0) // This can happen because of expiration but RMW operation haven't applied yet ++ { ++ while (!RespWriteUtils.WriteNull(ref curr, end)) ++ ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); ++ _output.result1 = 0; ++ return; ++ } ++ ++ var index = RandomUtils.PickRandomIndex(count, seed); ++ var pair = ElementAt(index); + while (!RespWriteUtils.WriteBulkString(pair.Key, ref curr, end)) + ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); + countDone = 1; +@@ -262,26 +287,31 @@ namespace Garnet.server + var key = input.parseState.GetArgSliceByRef(i).SpanByte.ToByteArray(); + var value = input.parseState.GetArgSliceByRef(i + 1).SpanByte.ToByteArray(); + +- if (!hash.TryGetValue(key, out var hashValue)) ++ if (!TryGetValue(key, out var hashValue)) + { +- hash.Add(key, value); +- this.UpdateSize(key, value); ++ Add(key, value); + _output->result1++; + } +- else if ((hop == HashOperation.HSET || hop == HashOperation.HMSET) && hashValue != default && +- !hashValue.AsSpan().SequenceEqual(value)) ++ else if ((hop == HashOperation.HSET || hop == HashOperation.HMSET) && hashValue != default) + { +- hash[key] = value; +- // Skip overhead as existing item is getting replaced. +- this.Size += Utility.RoundUp(value.Length, IntPtr.Size) - +- Utility.RoundUp(hashValue.Length, IntPtr.Size); ++ Set(key, value); + } + } + } + ++ private void HashCollect(ref ObjectInput input, byte* output) ++ { ++ var _output = (ObjectOutputHeader*)output; ++ *_output = default; ++ ++ DeleteExpiredItems(); ++ ++ _output->result1 = 1; ++ } ++ + private void HashGetKeysOrValues(ref ObjectInput input, ref SpanByteAndMemory output) + { +- var count = hash.Count; ++ var count = Count(); + var op = input.header.HashOp; + + var isMemory = false; +@@ -297,8 +327,15 @@ namespace Garnet.server + while (!RespWriteUtils.WriteArrayLength(count, ref curr, end)) + ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); + ++ var isExpirable = HasExpirableItems(); ++ + foreach (var item in hash) + { ++ if (isExpirable && IsExpired(item.Key)) ++ { ++ continue; ++ } ++ + if (HashOperation.HKEYS == op) + { + while (!RespWriteUtils.WriteBulkString(item.Key, ref curr, end)) +@@ -343,7 +380,7 @@ namespace Garnet.server + var key = input.parseState.GetArgSliceByRef(0).SpanByte.ToByteArray(); + var incrSlice = input.parseState.GetArgSliceByRef(1); + +- var valueExists = hash.TryGetValue(key, out var value); ++ var valueExists = TryGetValue(key, out var value); + if (op == HashOperation.HINCRBY) + { + if (!NumUtils.TryParse(incrSlice.ReadOnlySpan, out int incr)) +@@ -376,15 +413,12 @@ namespace Garnet.server + resultSpan = resultSpan.Slice(0, bytesWritten); + + resultBytes = resultSpan.ToArray(); +- hash[key] = resultBytes; +- Size += Utility.RoundUp(resultBytes.Length, IntPtr.Size) - +- Utility.RoundUp(value.Length, IntPtr.Size); ++ SetWithoutPersist(key, resultBytes); + } + else + { + resultBytes = incrSlice.SpanByte.ToByteArray(); +- hash.Add(key, resultBytes); +- UpdateSize(key, resultBytes); ++ Add(key, resultBytes); + } + + while (!RespWriteUtils.WriteIntegerFromBytes(resultBytes, ref curr, end)) +@@ -417,15 +451,12 @@ namespace Garnet.server + result += incr; + + resultBytes = Encoding.ASCII.GetBytes(result.ToString(CultureInfo.InvariantCulture)); +- hash[key] = resultBytes; +- Size += Utility.RoundUp(resultBytes.Length, IntPtr.Size) - +- Utility.RoundUp(value.Length, IntPtr.Size); ++ SetWithoutPersist(key, resultBytes); + } + else + { + resultBytes = incrSlice.SpanByte.ToByteArray(); +- hash.Add(key, resultBytes); +- UpdateSize(key, resultBytes); ++ Add(key, resultBytes); + } + + while (!RespWriteUtils.WriteBulkString(resultBytes, ref curr, end)) +@@ -444,5 +475,138 @@ namespace Garnet.server + output.Length = (int)(curr - ptr); + } + } ++ ++ private void HashExpire(ref ObjectInput input, ref SpanByteAndMemory output) ++ { ++ var isMemory = false; ++ MemoryHandle ptrHandle = default; ++ var ptr = output.SpanByte.ToPointer(); ++ ++ var curr = ptr; ++ var end = curr + output.Length; ++ ++ ObjectOutputHeader _output = default; ++ try ++ { ++ DeleteExpiredItems(); ++ ++ var expireOption = (ExpireOption)input.arg1; ++ var expiration = input.parseState.GetLong(0); ++ var numFields = input.parseState.Count - 1; ++ while (!RespWriteUtils.WriteArrayLength(numFields, ref curr, end)) ++ ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); ++ ++ foreach (var item in input.parseState.Parameters.Slice(1)) ++ { ++ var result = SetExpiration(item.ToArray(), expiration, expireOption); ++ while (!RespWriteUtils.WriteInteger(result, ref curr, end)) ++ ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); ++ _output.result1++; ++ } ++ } ++ finally ++ { ++ while (!RespWriteUtils.WriteDirect(ref _output, ref curr, end)) ++ ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); ++ ++ if (isMemory) ptrHandle.Dispose(); ++ output.Length = (int)(curr - ptr); ++ } ++ } ++ ++ private void HashTimeToLive(ref ObjectInput input, ref SpanByteAndMemory output) ++ { ++ var isMemory = false; ++ MemoryHandle ptrHandle = default; ++ var ptr = output.SpanByte.ToPointer(); ++ ++ var curr = ptr; ++ var end = curr + output.Length; ++ ++ ObjectOutputHeader _output = default; ++ try ++ { ++ DeleteExpiredItems(); ++ ++ var isMilliseconds = input.arg1 == 1; ++ var isTimestamp = input.arg2 == 1; ++ var numFields = input.parseState.Count; ++ while (!RespWriteUtils.WriteArrayLength(numFields, ref curr, end)) ++ ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); ++ ++ foreach (var item in input.parseState.Parameters) ++ { ++ var result = GetExpiration(item.ToArray()); ++ ++ if (result >= 0) ++ { ++ if (isTimestamp && isMilliseconds) ++ { ++ result = ConvertUtils.UnixTimeInMillisecondsFromTicks(result); ++ } ++ else if (isTimestamp && !isMilliseconds) ++ { ++ result = ConvertUtils.UnixTimeInSecondsFromTicks(result); ++ } ++ else if (!isTimestamp && isMilliseconds) ++ { ++ result = ConvertUtils.MillisecondsFromDiffUtcNowTicks(result); ++ } ++ else if (!isTimestamp && !isMilliseconds) ++ { ++ result = ConvertUtils.SecondsFromDiffUtcNowTicks(result); ++ } ++ } ++ ++ while (!RespWriteUtils.WriteInteger(result, ref curr, end)) ++ ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); ++ _output.result1++; ++ } ++ } ++ finally ++ { ++ while (!RespWriteUtils.WriteDirect(ref _output, ref curr, end)) ++ ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); ++ ++ if (isMemory) ptrHandle.Dispose(); ++ output.Length = (int)(curr - ptr); ++ } ++ } ++ ++ private void HashPersist(ref ObjectInput input, ref SpanByteAndMemory output) ++ { ++ var isMemory = false; ++ MemoryHandle ptrHandle = default; ++ var ptr = output.SpanByte.ToPointer(); ++ ++ var curr = ptr; ++ var end = curr + output.Length; ++ ++ ObjectOutputHeader _output = default; ++ try ++ { ++ DeleteExpiredItems(); ++ ++ var numFields = input.parseState.Count; ++ while (!RespWriteUtils.WriteArrayLength(numFields, ref curr, end)) ++ ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); ++ ++ foreach (var item in input.parseState.Parameters) ++ { ++ var result = Persist(item.ToArray()); ++ while (!RespWriteUtils.WriteInteger(result, ref curr, end)) ++ ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); ++ _output.result1++; ++ } ++ } ++ finally ++ { ++ while (!RespWriteUtils.WriteDirect(ref _output, ref curr, end)) ++ ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); ++ ++ if (isMemory) ptrHandle.Dispose(); ++ output.Length = (int)(curr - ptr); ++ } ++ } + } + } +\ No newline at end of file +diff --git a/libs/server/Objects/Types/GarnetObject.cs b/libs/server/Objects/Types/GarnetObject.cs +index f5366dff..7474d547 100644 +--- a/libs/server/Objects/Types/GarnetObject.cs ++++ b/libs/server/Objects/Types/GarnetObject.cs +@@ -66,6 +66,12 @@ namespace Garnet.server + SetOperation.SPOP => false, + _ => true, + }, ++ GarnetObjectType.Hash => header.HashOp switch ++ { ++ HashOperation.HEXPIRE => false, ++ HashOperation.HCOLLECT => false, ++ _ => true, ++ }, + GarnetObjectType.Expire => false, + GarnetObjectType.PExpire => false, + GarnetObjectType.Persist => false, diff --git a/libs/host/Configuration/Options.cs b/libs/host/Configuration/Options.cs index f55bfb2229..4bff1cdf03 100644 --- a/libs/host/Configuration/Options.cs +++ b/libs/host/Configuration/Options.cs @@ -239,6 +239,10 @@ internal sealed class Options [Option("hcollect-freq", Required = false, HelpText = "Frequency in seconds for the background task to perform Hash collection. 0 = disabled. Hash collect is used to delete expired fields from hash without waiting for a write operation. Use the HCOLLECT API to collect on-demand.")] public int HashCollectFrequencySecs { get; set; } + [IntRangeValidation(0, int.MaxValue)] + [Option("zcollect-freq", Required = false, HelpText = "Frequency in seconds for the background task to perform Sorted Set collection. 0 = disabled. Sorted Set collect is used to delete expired members from Sorted Set without waiting for a write operation. Use the ZCOLLECT API to collect on-demand.")] + public int SortedSetCollectFrequencySecs { get; set; } + [Option("compaction-type", Required = false, HelpText = "Hybrid log compaction type. Value options: None - no compaction, Shift - shift begin address without compaction (data loss), Scan - scan old pages and move live records to tail (no data loss), Lookup - lookup each record in compaction range, for record liveness checking using hash chain (no data loss)")] public LogCompactionType CompactionType { get; set; } @@ -701,6 +705,7 @@ public GarnetServerOptions GetServerOptions(ILogger logger = null) AofSizeLimit = AofSizeLimit, CompactionFrequencySecs = CompactionFrequencySecs, HashCollectFrequencySecs = HashCollectFrequencySecs, + SortedSetCollectFrequencySecs = SortedSetCollectFrequencySecs, CompactionType = CompactionType, CompactionForceDelete = CompactionForceDelete.GetValueOrDefault(), CompactionMaxSegments = CompactionMaxSegments, diff --git a/libs/host/defaults.conf b/libs/host/defaults.conf index f6bc0cebe3..df099e45ea 100644 --- a/libs/host/defaults.conf +++ b/libs/host/defaults.conf @@ -165,6 +165,9 @@ /* Frequency in seconds for the background task to perform Hash collection. 0 = disabled. Hash collect is used to delete expired fields from hash without waiting for a write operation. Use the HCOLLECT API to collect on-demand. */ "HashCollectFrequencySecs" : 0, + /* Frequency in seconds for the background task to perform Sorted Set collection. 0 = disabled. Sorted Set collect is used to delete expired members from Sorted Set without waiting for a write operation. Use the HCOLLECT API to collect on-demand. */ + "SortedSetCollectFrequencySecs" : 0, + /* Hybrid log compaction type. Value options: */ /* None - no compaction */ /* Shift - shift begin address without compaction (data loss) */ diff --git a/libs/resources/RespCommandsDocs.json b/libs/resources/RespCommandsDocs.json index a22cf66e5b..dc1f16c3a1 100644 --- a/libs/resources/RespCommandsDocs.json +++ b/libs/resources/RespCommandsDocs.json @@ -7006,6 +7006,12 @@ } ] }, + { + "Command": "ZCOLLECT", + "Name": "ZCOLLECT", + "Summary": "Manually trigger deletion of expired members from memory for SortedSet", + "Group": "Hash" + }, { "Command": "ZDIFF", "Name": "ZDIFF", @@ -7067,6 +7073,201 @@ } ] }, + { + "Command": "ZEXPIRE", + "Name": "ZEXPIRE", + "Summary": "Set expiry for sorted set member using relative time to expire (seconds)", + "Group": "SortedSet", + "Complexity": "O(N) where N is the number of specified members", + "Arguments": [ + { + "TypeDiscriminator": "RespCommandKeyArgument", + "Name": "KEY", + "DisplayText": "key", + "Type": "Key", + "KeySpecIndex": 0 + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "SECONDS", + "DisplayText": "seconds", + "Type": "Integer" + }, + { + "TypeDiscriminator": "RespCommandContainerArgument", + "Name": "CONDITION", + "Type": "OneOf", + "ArgumentFlags": "Optional", + "Arguments": [ + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "NX", + "DisplayText": "nx", + "Type": "PureToken", + "Token": "NX" + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "XX", + "DisplayText": "xx", + "Type": "PureToken", + "Token": "XX" + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "GT", + "DisplayText": "gt", + "Type": "PureToken", + "Token": "GT" + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "LT", + "DisplayText": "lt", + "Type": "PureToken", + "Token": "LT" + } + ] + }, + { + "TypeDiscriminator": "RespCommandContainerArgument", + "Name": "MEMBERS", + "Type": "Block", + "Token": "MEMBERS", + "Arguments": [ + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "NUMMEMBERS", + "DisplayText": "nummembers", + "Type": "Integer" + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "MEMBER", + "DisplayText": "member", + "Type": "String", + "ArgumentFlags": "Multiple" + } + ] + } + ] + }, + { + "Command": "ZEXPIREAT", + "Name": "ZEXPIREAT", + "Summary": "Set expiry for sorted set member using an absolute Unix timestamp (seconds)", + "Group": "SortedSet", + "Complexity": "O(N) where N is the number of specified members", + "Arguments": [ + { + "TypeDiscriminator": "RespCommandKeyArgument", + "Name": "KEY", + "DisplayText": "key", + "Type": "Key", + "KeySpecIndex": 0 + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "UNIX-TIME-SECONDS", + "DisplayText": "unix-time-seconds", + "Type": "UnixTime" + }, + { + "TypeDiscriminator": "RespCommandContainerArgument", + "Name": "CONDITION", + "Type": "OneOf", + "ArgumentFlags": "Optional", + "Arguments": [ + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "NX", + "DisplayText": "nx", + "Type": "PureToken", + "Token": "NX" + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "XX", + "DisplayText": "xx", + "Type": "PureToken", + "Token": "XX" + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "GT", + "DisplayText": "gt", + "Type": "PureToken", + "Token": "GT" + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "LT", + "DisplayText": "lt", + "Type": "PureToken", + "Token": "LT" + } + ] + }, + { + "TypeDiscriminator": "RespCommandContainerArgument", + "Name": "MEMBERS", + "Type": "Block", + "Token": "MEMBERS", + "Arguments": [ + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "NUMMEMBERS", + "DisplayText": "nummembers", + "Type": "Integer" + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "MEMBER", + "DisplayText": "member", + "Type": "String", + "ArgumentFlags": "Multiple" + } + ] + } + ] + }, + { + "Command": "ZEXPIRETIME", + "Name": "ZEXPIRETIME", + "Summary": "Returns the expiration time of a sorted set member as a Unix timestamp, in seconds.", + "Group": "SortedSet", + "Complexity": "O(N) where N is the number of specified members", + "Arguments": [ + { + "TypeDiscriminator": "RespCommandKeyArgument", + "Name": "KEY", + "DisplayText": "key", + "Type": "Key", + "KeySpecIndex": 0 + }, + { + "TypeDiscriminator": "RespCommandContainerArgument", + "Name": "FIELDS", + "Type": "Block", + "Token": "FIELDS", + "Arguments": [ + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "NUMFIELDS", + "DisplayText": "numfields", + "Type": "Integer" + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "FIELD", + "DisplayText": "field", + "Type": "String", + "ArgumentFlags": "Multiple" + } + ] + } + ] + }, { "Command": "ZINCRBY", "Name": "ZINCRBY", @@ -7366,6 +7567,275 @@ } ] }, + { + "Command": "ZPERSIST", + "Name": "ZPERSIST", + "Summary": "Removes the expiration time for each specified sorted set member", + "Group": "SortedSet", + "Complexity": "O(N) where N is the number of specified members", + "Arguments": [ + { + "TypeDiscriminator": "RespCommandKeyArgument", + "Name": "KEY", + "DisplayText": "key", + "Type": "Key", + "KeySpecIndex": 0 + }, + { + "TypeDiscriminator": "RespCommandContainerArgument", + "Name": "MEMBERS", + "Type": "Block", + "Token": "MEMBERS", + "Arguments": [ + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "NUMMEMBERS", + "DisplayText": "nummembers", + "Type": "Integer" + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "MEMBER", + "DisplayText": "member", + "Type": "String", + "ArgumentFlags": "Multiple" + } + ] + } + ] + }, + { + "Command": "ZPEXPIRE", + "Name": "ZPEXPIRE", + "Summary": "Set expiry for sorted set member using relative time to expire (milliseconds)", + "Group": "SortedSet", + "Complexity": "O(N) where N is the number of specified members", + "Arguments": [ + { + "TypeDiscriminator": "RespCommandKeyArgument", + "Name": "KEY", + "DisplayText": "key", + "Type": "Key", + "KeySpecIndex": 0 + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "MILLISECONDS", + "DisplayText": "milliseconds", + "Type": "Integer" + }, + { + "TypeDiscriminator": "RespCommandContainerArgument", + "Name": "CONDITION", + "Type": "OneOf", + "ArgumentFlags": "Optional", + "Arguments": [ + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "NX", + "DisplayText": "nx", + "Type": "PureToken", + "Token": "NX" + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "XX", + "DisplayText": "xx", + "Type": "PureToken", + "Token": "XX" + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "GT", + "DisplayText": "gt", + "Type": "PureToken", + "Token": "GT" + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "LT", + "DisplayText": "lt", + "Type": "PureToken", + "Token": "LT" + } + ] + }, + { + "TypeDiscriminator": "RespCommandContainerArgument", + "Name": "MEMBERS", + "Type": "Block", + "Token": "MEMBERS", + "Arguments": [ + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "NUMMEMBERS", + "DisplayText": "nummembers", + "Type": "Integer" + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "MEMBER", + "DisplayText": "member", + "Type": "String", + "ArgumentFlags": "Multiple" + } + ] + } + ] + }, + { + "Command": "ZPEXPIREAT", + "Name": "ZPEXPIREAT", + "Summary": "Set expiry for sorted set member using an absolute Unix timestamp (milliseconds)", + "Group": "SortedSet", + "Complexity": "O(N) where N is the number of specified members", + "Arguments": [ + { + "TypeDiscriminator": "RespCommandKeyArgument", + "Name": "KEY", + "DisplayText": "key", + "Type": "Key", + "KeySpecIndex": 0 + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "UNIX-TIME-MILLISECONDS", + "DisplayText": "unix-time-milliseconds", + "Type": "UnixTime" + }, + { + "TypeDiscriminator": "RespCommandContainerArgument", + "Name": "CONDITION", + "Type": "OneOf", + "ArgumentFlags": "Optional", + "Arguments": [ + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "NX", + "DisplayText": "nx", + "Type": "PureToken", + "Token": "NX" + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "XX", + "DisplayText": "xx", + "Type": "PureToken", + "Token": "XX" + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "GT", + "DisplayText": "gt", + "Type": "PureToken", + "Token": "GT" + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "LT", + "DisplayText": "lt", + "Type": "PureToken", + "Token": "LT" + } + ] + }, + { + "TypeDiscriminator": "RespCommandContainerArgument", + "Name": "MEMBERS", + "Type": "Block", + "Token": "MEMBERS", + "Arguments": [ + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "NUMMEMBERS", + "DisplayText": "nummembers", + "Type": "Integer" + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "MEMBER", + "DisplayText": "member", + "Type": "String", + "ArgumentFlags": "Multiple" + } + ] + } + ] + }, + { + "Command": "ZPEXPIRETIME", + "Name": "ZPEXPIRETIME", + "Summary": "Returns the expiration time of a sorted set member as a Unix timestamp, in msec.", + "Group": "SortedSet", + "Complexity": "O(N) where N is the number of specified members", + "Arguments": [ + { + "TypeDiscriminator": "RespCommandKeyArgument", + "Name": "KEY", + "DisplayText": "key", + "Type": "Key", + "KeySpecIndex": 0 + }, + { + "TypeDiscriminator": "RespCommandContainerArgument", + "Name": "MEMBERS", + "Type": "Block", + "Token": "MEMBERS", + "Arguments": [ + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "NUMMEMBERS", + "DisplayText": "nummembers", + "Type": "Integer" + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "MEMBER", + "DisplayText": "member", + "Type": "String", + "ArgumentFlags": "Multiple" + } + ] + } + ] + }, + { + "Command": "ZPTTL", + "Name": "ZPTTL", + "Summary": "Returns the TTL in milliseconds of a sorted set member.", + "Group": "SortedSet", + "Complexity": "O(N) where N is the number of specified members", + "Arguments": [ + { + "TypeDiscriminator": "RespCommandKeyArgument", + "Name": "KEY", + "DisplayText": "key", + "Type": "Key", + "KeySpecIndex": 0 + }, + { + "TypeDiscriminator": "RespCommandContainerArgument", + "Name": "MEMBERS", + "Type": "Block", + "Token": "MEMBERS", + "Arguments": [ + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "NUMMEMBERS", + "DisplayText": "nummembers", + "Type": "Integer" + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "MEMBER", + "DisplayText": "member", + "Type": "String", + "ArgumentFlags": "Multiple" + } + ] + } + ] + }, { "Command": "ZPOPMAX", "Name": "ZPOPMAX", @@ -8057,6 +8527,43 @@ } ] }, + { + "Command": "ZTTL", + "Name": "ZTTL", + "Summary": "Returns the TTL in seconds of a sorted set member.", + "Group": "SortedSet", + "Complexity": "O(N) where N is the number of specified members", + "Arguments": [ + { + "TypeDiscriminator": "RespCommandKeyArgument", + "Name": "KEY", + "DisplayText": "key", + "Type": "Key", + "KeySpecIndex": 0 + }, + { + "TypeDiscriminator": "RespCommandContainerArgument", + "Name": "MEMBERS", + "Type": "Block", + "Token": "MEMBERS", + "Arguments": [ + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "NUMMEMBERS", + "DisplayText": "nummembers", + "Type": "Integer" + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "MEMBER", + "DisplayText": "member", + "Type": "String", + "ArgumentFlags": "Multiple" + } + ] + } + ] + }, { "Command": "ZUNION", "Name": "ZUNION", diff --git a/libs/resources/RespCommandsInfo.json b/libs/resources/RespCommandsInfo.json index 159b575b5a..9b0460e064 100644 --- a/libs/resources/RespCommandsInfo.json +++ b/libs/resources/RespCommandsInfo.json @@ -4788,6 +4788,31 @@ } ] }, + { + "Command": "ZCOLLECT", + "Name": "ZCOLLECT", + "Arity": 2, + "Flags": "Admin, Write", + "FirstKey": 1, + "LastKey": 1, + "Step": 1, + "AclCategories": "SortedSet, Write, Admin, Garnet", + "KeySpecifications": [ + { + "BeginSearch": { + "TypeDiscriminator": "BeginSearchIndex", + "Index": 1 + }, + "FindKeys": { + "TypeDiscriminator": "FindKeysRange", + "LastKey": 0, + "KeyStep": 1, + "Limit": 0 + }, + "Flags": "RW, Access, Update" + } + ] + }, { "Command": "ZDIFF", "Name": "ZDIFF", @@ -4848,6 +4873,81 @@ } ] }, + { + "Command": "ZEXPIRE", + "Name": "ZEXPIRE", + "Arity": -6, + "Flags": "DenyOom, Fast, Write", + "FirstKey": 1, + "LastKey": 1, + "Step": 1, + "AclCategories": "SortedSet, Fast, Write, Garnet", + "KeySpecifications": [ + { + "BeginSearch": { + "TypeDiscriminator": "BeginSearchIndex", + "Index": 1 + }, + "FindKeys": { + "TypeDiscriminator": "FindKeysRange", + "LastKey": 0, + "KeyStep": 1, + "Limit": 0 + }, + "Flags": "RW, Update" + } + ] + }, + { + "Command": "ZEXPIREAT", + "Name": "ZEXPIREAT", + "Arity": -6, + "Flags": "DenyOom, Fast, Write", + "FirstKey": 1, + "LastKey": 1, + "Step": 1, + "AclCategories": "SortedSet, Fast, Write, Garnet", + "KeySpecifications": [ + { + "BeginSearch": { + "TypeDiscriminator": "BeginSearchIndex", + "Index": 1 + }, + "FindKeys": { + "TypeDiscriminator": "FindKeysRange", + "LastKey": 0, + "KeyStep": 1, + "Limit": 0 + }, + "Flags": "RW, Update" + } + ] + }, + { + "Command": "ZEXPIRETIME", + "Name": "ZEXPIRETIME", + "Arity": -5, + "Flags": "Fast, ReadOnly", + "FirstKey": 1, + "LastKey": 1, + "Step": 1, + "AclCategories": "SortedSet, Fast, Read, Garnet", + "KeySpecifications": [ + { + "BeginSearch": { + "TypeDiscriminator": "BeginSearchIndex", + "Index": 1 + }, + "FindKeys": { + "TypeDiscriminator": "FindKeysRange", + "LastKey": 0, + "KeyStep": 1, + "Limit": 0 + }, + "Flags": "RO, Access" + } + ] + }, { "Command": "ZINCRBY", "Name": "ZINCRBY", @@ -5027,6 +5127,131 @@ } ] }, + { + "Command": "ZPERSIST", + "Name": "ZPERSIST", + "Arity": -5, + "Flags": "Fast, Write", + "FirstKey": 1, + "LastKey": 1, + "Step": 1, + "AclCategories": "SortedSet, Fast, Write, Garnet", + "KeySpecifications": [ + { + "BeginSearch": { + "TypeDiscriminator": "BeginSearchIndex", + "Index": 1 + }, + "FindKeys": { + "TypeDiscriminator": "FindKeysRange", + "LastKey": 0, + "KeyStep": 1, + "Limit": 0 + }, + "Flags": "RW, Update" + } + ] + }, + { + "Command": "ZPEXPIRE", + "Name": "ZPEXPIRE", + "Arity": -6, + "Flags": "DenyOom, Fast, Write", + "FirstKey": 1, + "LastKey": 1, + "Step": 1, + "AclCategories": "SortedSet, Fast, Write, Garnet", + "KeySpecifications": [ + { + "BeginSearch": { + "TypeDiscriminator": "BeginSearchIndex", + "Index": 1 + }, + "FindKeys": { + "TypeDiscriminator": "FindKeysRange", + "LastKey": 0, + "KeyStep": 1, + "Limit": 0 + }, + "Flags": "RW, Update" + } + ] + }, + { + "Command": "ZPEXPIREAT", + "Name": "ZPEXPIREAT", + "Arity": -6, + "Flags": "DenyOom, Fast, Write", + "FirstKey": 1, + "LastKey": 1, + "Step": 1, + "AclCategories": "SortedSet, Fast, Write, Garnet", + "KeySpecifications": [ + { + "BeginSearch": { + "TypeDiscriminator": "BeginSearchIndex", + "Index": 1 + }, + "FindKeys": { + "TypeDiscriminator": "FindKeysRange", + "LastKey": 0, + "KeyStep": 1, + "Limit": 0 + }, + "Flags": "RW, Update" + } + ] + }, + { + "Command": "ZPEXPIRETIME", + "Name": "ZPEXPIRETIME", + "Arity": -5, + "Flags": "Fast, ReadOnly", + "FirstKey": 1, + "LastKey": 1, + "Step": 1, + "AclCategories": "SortedSet, Fast, Read, Garnet", + "KeySpecifications": [ + { + "BeginSearch": { + "TypeDiscriminator": "BeginSearchIndex", + "Index": 1 + }, + "FindKeys": { + "TypeDiscriminator": "FindKeysRange", + "LastKey": 0, + "KeyStep": 1, + "Limit": 0 + }, + "Flags": "RO, Access" + } + ] + }, + { + "Command": "ZPTTL", + "Name": "ZPTTL", + "Arity": -5, + "Flags": "Fast, ReadOnly", + "FirstKey": 1, + "LastKey": 1, + "Step": 1, + "AclCategories": "SortedSet, Fast, Read, Garnet", + "KeySpecifications": [ + { + "BeginSearch": { + "TypeDiscriminator": "BeginSearchIndex", + "Index": 1 + }, + "FindKeys": { + "TypeDiscriminator": "FindKeysRange", + "LastKey": 0, + "KeyStep": 1, + "Limit": 0 + }, + "Flags": "RO, Access" + } + ] + }, { "Command": "ZPOPMAX", "Name": "ZPOPMAX", @@ -5446,6 +5671,31 @@ } ] }, + { + "Command": "ZTTL", + "Name": "ZTTL", + "Arity": -5, + "Flags": "Fast, ReadOnly", + "FirstKey": 1, + "LastKey": 1, + "Step": 1, + "AclCategories": "SortedSet, Fast, Read, Garnet", + "KeySpecifications": [ + { + "BeginSearch": { + "TypeDiscriminator": "BeginSearchIndex", + "Index": 1 + }, + "FindKeys": { + "TypeDiscriminator": "FindKeysRange", + "LastKey": 0, + "KeyStep": 1, + "Limit": 0 + }, + "Flags": "RO, Access" + } + ] + }, { "Command": "ZSCORE", "Name": "ZSCORE", diff --git a/libs/server/API/GarnetApiObjectCommands.cs b/libs/server/API/GarnetApiObjectCommands.cs index d07988a211..c079fe5858 100644 --- a/libs/server/API/GarnetApiObjectCommands.cs +++ b/libs/server/API/GarnetApiObjectCommands.cs @@ -169,6 +169,22 @@ public GarnetStatus SortedSetIntersectLength(ReadOnlySpan keys, int? l public GarnetStatus SortedSetIntersectStore(ArgSlice destinationKey, ReadOnlySpan keys, double[] weights, SortedSetAggregateType aggregateType, out int count) => storageSession.SortedSetIntersectStore(destinationKey, keys, weights, aggregateType, out count); + /// + public GarnetStatus SortedSetExpire(ArgSlice key, long expireAt, bool isMilliseconds, ExpireOption expireOption, ref ObjectInput input, ref GarnetObjectStoreOutput outputFooter) + => storageSession.SortedSetExpire(key, expireAt, isMilliseconds, expireOption, ref input, ref outputFooter, ref objectContext); + + /// + public GarnetStatus SortedSetPersist(ArgSlice key, ref ObjectInput input, ref GarnetObjectStoreOutput outputFooter) + => storageSession.SortedSetPersist(key, ref input, ref outputFooter, ref objectContext); + + /// + public GarnetStatus SortedSetTimeToLive(ArgSlice key, bool isMilliseconds, bool isTimestamp, ref ObjectInput input, ref GarnetObjectStoreOutput outputFooter) + => storageSession.SortedSetTimeToLive(key, isMilliseconds, isTimestamp, ref input, ref outputFooter, ref objectContext); + + /// + public GarnetStatus SortedSetCollect(ReadOnlySpan keys, ref ObjectInput input) + => storageSession.SortedSetCollect(keys, ref input, ref objectContext); + #endregion #region Geospatial commands diff --git a/libs/server/API/GarnetWatchApi.cs b/libs/server/API/GarnetWatchApi.cs index e95c491e34..3dc0e2db68 100644 --- a/libs/server/API/GarnetWatchApi.cs +++ b/libs/server/API/GarnetWatchApi.cs @@ -243,6 +243,13 @@ public GarnetStatus SortedSetIntersectLength(ReadOnlySpan keys, int? l return garnetApi.SortedSetIntersectLength(keys, limit, out count); } + /// + public GarnetStatus SortedSetTimeToLive(ArgSlice key, bool isMilliseconds, bool isTimestamp, ref ObjectInput input, ref GarnetObjectStoreOutput outputFooter) + { + garnetApi.WATCH(key, StoreType.Object); + return garnetApi.SortedSetTimeToLive(key, isMilliseconds, isTimestamp, ref input, ref outputFooter); + } + #endregion #region List Methods diff --git a/libs/server/API/IGarnetApi.cs b/libs/server/API/IGarnetApi.cs index 249f59d8df..d88ee8d62f 100644 --- a/libs/server/API/IGarnetApi.cs +++ b/libs/server/API/IGarnetApi.cs @@ -543,6 +543,34 @@ public interface IGarnetApi : IGarnetReadApi, IGarnetAdvancedApi /// A indicating the status of the operation. GarnetStatus SortedSetUnionStore(ArgSlice destinationKey, ReadOnlySpan keys, double[] weights, SortedSetAggregateType aggregateType, out int count); + /// + /// Sets an expiration time on a sorted set member. + /// + /// The key of the sorted set. + /// The expiration time in Unix timestamp format. + /// The expiration option to apply. + /// The input object containing additional parameters. + /// The output object to store the result. + /// The status of the operation. + GarnetStatus SortedSetExpire(ArgSlice key, long expireAt, bool isMilliseconds, ExpireOption expireOption, ref ObjectInput input, ref GarnetObjectStoreOutput outputFooter); + + /// + /// Persists the specified sorted set member, removing any expiration time set on it. + /// + /// The key of the sorted set to persist. + /// The input object containing additional parameters. + /// The output object to store the result. + /// The status of the operation. + GarnetStatus SortedSetPersist(ArgSlice key, ref ObjectInput input, ref GarnetObjectStoreOutput outputFooter); + + /// + /// Deletes already expired members from the sorted set. + /// + /// The keys of the sorted set members to check for expiration. + /// The input object containing additional parameters. + /// The status of the operation. + GarnetStatus SortedSetCollect(ReadOnlySpan keys, ref ObjectInput input); + #endregion #region Set Methods @@ -1397,6 +1425,17 @@ public interface IGarnetReadApi /// Operation status GarnetStatus SortedSetIntersectLength(ReadOnlySpan keys, int? limit, out int count); + /// + /// Returns the time to live for a sorted set members. + /// + /// The key of the sorted set. + /// Indicates if the time to live is in milliseconds. + /// Indicates if the time to live is a timestamp. + /// The input object containing additional parameters. + /// The output object to store the result. + /// The status of the operation. + GarnetStatus SortedSetTimeToLive(ArgSlice key, bool isMilliseconds, bool isTimestamp, ref ObjectInput input, ref GarnetObjectStoreOutput outputFooter); + #endregion #region Geospatial Methods diff --git a/libs/server/Objects/SortedSet/SortedSetObject.cs b/libs/server/Objects/SortedSet/SortedSetObject.cs index cbfa9dd1a6..960886152e 100644 --- a/libs/server/Objects/SortedSet/SortedSetObject.cs +++ b/libs/server/Objects/SortedSet/SortedSetObject.cs @@ -6,6 +6,8 @@ using System.Collections.Generic; using System.Diagnostics; using System.IO; +using System.Linq; +using System.Runtime.CompilerServices; using Garnet.common; using Tsavorite.core; @@ -47,7 +49,11 @@ public enum SortedSetOperation : byte ZRANDMEMBER, ZDIFF, ZSCAN, - ZMSCORE + ZMSCORE, + ZEXPIRE, + ZTTL, + ZPERSIST, + ZCOLLECT } [Flags] @@ -105,10 +111,15 @@ public enum SortedSetOrderOperation /// /// Sorted Set /// - public partial class SortedSetObject : GarnetObjectBase + public unsafe partial class SortedSetObject : GarnetObjectBase { private readonly SortedSet<(double Score, byte[] Element)> sortedSet; private readonly Dictionary sortedSetDict; + private Dictionary expirationTimes; + private PriorityQueue expirationQueue; + + // Byte #31 is used to denote if key has expiration (1) or not (0) + private const int ExpirationBitMask = 1 << 31; /// /// Constructor @@ -132,10 +143,31 @@ public SortedSetObject(BinaryReader reader) int count = reader.ReadInt32(); for (int i = 0; i < count; i++) { - var item = reader.ReadBytes(reader.ReadInt32()); + var keyLength = reader.ReadInt32(); + var hasExpiration = (keyLength & ExpirationBitMask) != 0; + keyLength &= ~ExpirationBitMask; + var item = reader.ReadBytes(keyLength); var score = reader.ReadDouble(); - sortedSet.Add((score, item)); - sortedSetDict.Add(item, score); + + if (hasExpiration) + { + var expiration = reader.ReadInt64(); + var isExpired = expiration < DateTimeOffset.UtcNow.Ticks; + if (!isExpired) + { + sortedSetDict.Add(item, score); + sortedSet.Add((score, item)); + InitializeExpirationStructures(); + expirationTimes.Add(item, expiration); + expirationQueue.Enqueue(item, expiration); + UpdateExpirationSize(item, true); + } + } + else + { + sortedSetDict.Add(item, score); + sortedSet.Add((score, item)); + } this.UpdateSize(item); } @@ -144,11 +176,13 @@ public SortedSetObject(BinaryReader reader) /// /// Copy constructor /// - public SortedSetObject(SortedSet<(double, byte[])> sortedSet, Dictionary sortedSetDict, long expiration, long size) + public SortedSetObject(SortedSet<(double, byte[])> sortedSet, Dictionary sortedSetDict, Dictionary expirationTimes, PriorityQueue expirationQueue, long expiration, long size) : base(expiration, size) { this.sortedSet = sortedSet; this.sortedSetDict = sortedSetDict; + this.expirationTimes = expirationTimes; + this.expirationQueue = expirationQueue; } /// @@ -166,10 +200,22 @@ public override void DoSerialize(BinaryWriter writer) { base.DoSerialize(writer); - int count = sortedSetDict.Count; + DeleteExpiredItems(); + + int count = sortedSetDict.Count; // Since expired items are already deleted, no need to worry about expiring items writer.Write(count); foreach (var kvp in sortedSetDict) { + if (expirationTimes is not null && expirationTimes.TryGetValue(kvp.Key, out var expiration)) + { + writer.Write(kvp.Key.Length | ExpirationBitMask); + writer.Write(kvp.Key); + writer.Write(kvp.Value); + writer.Write(expiration); + count--; + continue; + } + writer.Write(kvp.Key.Length); writer.Write(kvp.Key); writer.Write(kvp.Value); @@ -185,6 +231,8 @@ public override void DoSerialize(BinaryWriter writer) /// public void Add(byte[] item, double score) { + DeleteExpiredItems(); + sortedSetDict.Add(item, score); sortedSet.Add((score, item)); @@ -196,6 +244,7 @@ public void Add(byte[] item, double score) /// public bool Equals(SortedSetObject other) { + // TODO: Implement equals with expiration times if (sortedSetDict.Count != other.sortedSetDict.Count) return false; foreach (var key in sortedSetDict) @@ -209,7 +258,7 @@ public bool Equals(SortedSetObject other) public override void Dispose() { } /// - public override GarnetObjectBase Clone() => new SortedSetObject(sortedSet, sortedSetDict, Expiration, Size); + public override GarnetObjectBase Clone() => new SortedSetObject(sortedSet, sortedSetDict, expirationTimes, expirationQueue, Expiration, Size); /// public override unsafe bool Operate(ref ObjectInput input, ref GarnetObjectStoreOutput output, out long sizeChange) @@ -265,6 +314,18 @@ public override unsafe bool Operate(ref ObjectInput input, ref GarnetObjectStore case SortedSetOperation.ZRANGEBYSCORE: SortedSetRange(ref input, ref output.SpanByteAndMemory); break; + case SortedSetOperation.ZEXPIRE: + SortedSetExpire(ref input, ref output.SpanByteAndMemory); + break; + case SortedSetOperation.ZTTL: + SortedSetTimeToLive(ref input, ref output.SpanByteAndMemory); + break; + case SortedSetOperation.ZPERSIST: + SortedSetPersist(ref input, ref output.SpanByteAndMemory); + break; + case SortedSetOperation.ZCOLLECT: + SortedSetCollect(ref input, outputSpan); + break; case SortedSetOperation.GEOADD: GeoAdd(ref input, ref output.SpanByteAndMemory); break; @@ -355,8 +416,15 @@ public override unsafe void Scan(long start, out List items, out long cu return; } + var expiredKeysCount = 0; foreach (var item in Dictionary) { + if (IsExpired(item.Key)) + { + expiredKeysCount++; + continue; + } + if (index < start) { index++; @@ -368,7 +436,7 @@ public override unsafe void Scan(long start, out List items, out long cu { items.Add(item.Key); addToList = true; - } + } else { fixed (byte* keyPtr = item.Key) @@ -377,9 +445,9 @@ public override unsafe void Scan(long start, out List items, out long cu { items.Add(item.Key); addToList = true; + } } } - } if (addToList) { @@ -398,7 +466,7 @@ public override unsafe void Scan(long start, out List items, out long cu } // Indicates end of collection has been reached. - if (cursor == Dictionary.Count) + if (cursor + expiredKeysCount == sortedSetDict.Count) cursor = 0; } @@ -443,6 +511,210 @@ public static void InPlaceDiff(Dictionary dict1, Dictionary(ByteArrayComparer.Instance); + expirationQueue = new PriorityQueue(); + this.Size += MemoryUtils.DictionaryOverhead + MemoryUtils.PriorityQueueOverhead; + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private void UpdateExpirationSize(ReadOnlySpan key, bool add = true) + { + var size = IntPtr.Size + sizeof(long) + MemoryUtils.DictionaryEntryOverhead + + IntPtr.Size + sizeof(long) + MemoryUtils.PriorityQueueEntryOverhead; + this.Size += add ? size : -size; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private void CleanupExpirationStructures() + { + if (expirationTimes.Count == 0) + { + this.Size -= (IntPtr.Size + sizeof(long) + MemoryUtils.PriorityQueueOverhead) * expirationQueue.Count; + this.Size -= MemoryUtils.DictionaryOverhead + MemoryUtils.PriorityQueueOverhead; + expirationTimes = null; + expirationQueue = null; + } + } + + private void DeleteExpiredItems() + { + if (expirationTimes is null) + return; + + while (expirationQueue.TryPeek(out var key, out var expiration) && expiration < DateTimeOffset.UtcNow.Ticks) + { + if (expirationTimes.TryGetValue(key, out var actualExpiration) && actualExpiration == expiration) + { + expirationTimes.Remove(key); + expirationQueue.Dequeue(); + UpdateExpirationSize(key, false); + if (sortedSetDict.TryGetValue(key, out var value)) + { + sortedSetDict.Remove(key); + sortedSet.Remove((value, key)); + UpdateSize(key, false); + } + } + else + { + expirationQueue.Dequeue(); + this.Size -= MemoryUtils.PriorityQueueEntryOverhead + IntPtr.Size + sizeof(long); + } + } + + CleanupExpirationStructures(); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private bool TryGetScore(byte[] key, out double value) + { + value = default; + if (IsExpired(key)) + { + return false; + } + + return sortedSetDict.TryGetValue(key, out value); + } + + private int Count() + { + if (expirationTimes is null) + { + return sortedSetDict.Count; + } + var expiredKeysCount = 0; + foreach (var item in expirationTimes) + { + if (IsExpired(item.Key)) + { + expiredKeysCount++; + } + } + return sortedSetDict.Count - expiredKeysCount; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private bool IsExpired(byte[] key) => expirationTimes is not null && expirationTimes.TryGetValue(key, out var expiration) && expiration < DateTimeOffset.UtcNow.Ticks; + + private int SetExpiration(byte[] key, long expiration, ExpireOption expireOption) + { + if (!sortedSetDict.ContainsKey(key)) + { + return (int)ExpireResult.KeyNotFound; + } + + if (expiration <= DateTimeOffset.UtcNow.Ticks) + { + sortedSetDict.Remove(key, out var value); + sortedSet.Remove((value, key)); + UpdateSize(key, false); + return (int)ExpireResult.KeyAlreadyExpired; + } + + InitializeExpirationStructures(); + + if (expirationTimes.TryGetValue(key, out var currentExpiration)) + { + if (expireOption.HasFlag(ExpireOption.NX) || + (expireOption.HasFlag(ExpireOption.GT) && expiration <= currentExpiration) || + (expireOption.HasFlag(ExpireOption.LT) && expiration >= currentExpiration)) + { + return (int)ExpireResult.ExpireConditionNotMet; + } + + expirationTimes[key] = expiration; + expirationQueue.Enqueue(key, expiration); + this.Size += IntPtr.Size + sizeof(long) + MemoryUtils.PriorityQueueEntryOverhead; + } + else + { + if (expireOption.HasFlag(ExpireOption.XX) || expireOption.HasFlag(ExpireOption.GT)) + { + return (int)ExpireResult.ExpireConditionNotMet; + } + + expirationTimes[key] = expiration; + expirationQueue.Enqueue(key, expiration); + UpdateExpirationSize(key); + } + + return (int)ExpireResult.ExpireUpdated; + } + + private int Persist(byte[] key) + { + if (!sortedSetDict.ContainsKey(key)) + { + return -2; + } + + return TryRemoveExpiration(key) ? 1 : - 1; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private bool TryRemoveExpiration(byte[] key) + { + if (expirationTimes is null || !expirationTimes.TryGetValue(key, out _)) + { + return false; + } + + expirationTimes.Remove(key); + this.Size -= IntPtr.Size + sizeof(long) + MemoryUtils.DictionaryEntryOverhead; + CleanupExpirationStructures(); + return true; + } + + private long GetExpiration(byte[] key) + { + if (!sortedSetDict.ContainsKey(key)) + { + return -2; + } + + if (expirationTimes is not null && expirationTimes.TryGetValue(key, out var expiration)) + { + return expiration; + } + + return -1; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private bool HasExpirableItems() + { + return expirationTimes is not null; + } + + private KeyValuePair ElementAt(int index) + { + if (HasExpirableItems()) + { + var currIndex = 0; + foreach (var item in sortedSetDict) + { + if (IsExpired(item.Key)) + { + continue; + } + + if (currIndex++ == index) + { + return item; + } + } + + throw new ArgumentOutOfRangeException("index is outside the bounds of the source sequence."); + } + + return sortedSetDict.ElementAt(index); + } private void UpdateSize(ReadOnlySpan item, bool add = true) { @@ -452,5 +724,13 @@ private void UpdateSize(ReadOnlySpan item, bool add = true) this.Size += add ? size : -size; Debug.Assert(this.Size >= MemoryUtils.SortedSetOverhead + MemoryUtils.DictionaryOverhead); } + + enum ExpireResult + { + KeyNotFound = -2, + ExpireConditionNotMet = 0, + ExpireUpdated = 1, + KeyAlreadyExpired = 2, + } } } \ No newline at end of file diff --git a/libs/server/Objects/SortedSet/SortedSetObjectImpl.cs b/libs/server/Objects/SortedSet/SortedSetObjectImpl.cs index f22b7c68c7..4bbb472c65 100644 --- a/libs/server/Objects/SortedSet/SortedSetObjectImpl.cs +++ b/libs/server/Objects/SortedSet/SortedSetObjectImpl.cs @@ -85,6 +85,8 @@ bool GetOptions(ref ObjectInput input, ref int currTokenIdx, out SortedSetAddOpt private void SortedSetAdd(ref ObjectInput input, ref SpanByteAndMemory output) { + DeleteExpiredItems(); + var isMemory = false; MemoryHandle ptrHandle = default; var ptr = output.SpanByte.ToPointer(); @@ -167,6 +169,7 @@ private void SortedSetAdd(ref ObjectInput input, ref SpanByteAndMemory output) var success = sortedSet.Remove((scoreStored, member)); Debug.Assert(success); success = sortedSet.Add((score, member)); + Persist(member); Debug.Assert(success); // If CH flag is set, add changed member to final count @@ -198,6 +201,8 @@ private void SortedSetAdd(ref ObjectInput input, ref SpanByteAndMemory output) private void SortedSetRemove(ref ObjectInput input, byte* output) { + DeleteExpiredItems(); + var _output = (ObjectOutputHeader*)output; *_output = default; @@ -212,6 +217,7 @@ private void SortedSetRemove(ref ObjectInput input, byte* output) _output->result1++; sortedSetDict.Remove(valueArray); sortedSet.Remove((key, valueArray)); + TryRemoveExpiration(valueArray); this.UpdateSize(value, false); } @@ -221,7 +227,7 @@ private void SortedSetLength(byte* output) { // Check both objects Debug.Assert(sortedSetDict.Count == sortedSet.Count, "SortedSet object is not in sync."); - ((ObjectOutputHeader*)output)->result1 = sortedSetDict.Count; + ((ObjectOutputHeader*)output)->result1 = Count(); } private void SortedSetScore(ref ObjectInput input, ref SpanByteAndMemory output) @@ -239,7 +245,7 @@ private void SortedSetScore(ref ObjectInput input, ref SpanByteAndMemory output) ObjectOutputHeader outputHeader = default; try { - if (!sortedSetDict.TryGetValue(member, out var score)) + if (!TryGetScore(member, out var score)) { while (!RespWriteUtils.TryWriteNull(ref curr, end)) ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); @@ -284,7 +290,7 @@ private void SortedSetScores(ref ObjectInput input, ref SpanByteAndMemory output { var member = input.parseState.GetArgSliceByRef(i).SpanByte.ToByteArray(); - if (!sortedSetDict.TryGetValue(member, out var score)) + if (!TryGetScore(member, out var score)) { while (!RespWriteUtils.TryWriteNull(ref curr, end)) ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); @@ -339,6 +345,7 @@ private void SortedSetCount(ref ObjectInput input, ref SpanByteAndMemory output) { foreach (var item in sortedSet.GetViewBetween((minValue, null), sortedSet.Max)) { + if (IsExpired(item.Element)) continue; if (item.Item1 > maxValue || (maxExclusive && item.Item1 == maxValue)) break; if (minExclusive && item.Item1 == minValue) continue; count++; @@ -385,7 +392,8 @@ private void SortedSetIncrement(ref ObjectInput input, ref SpanByteAndMemory out if (sortedSetDict.TryGetValue(member, out var score)) { - sortedSetDict[member] += incrValue; + score = IsExpired(member) ? 0 : score; + sortedSetDict[member] = score + incrValue; sortedSet.Remove((score, member)); sortedSet.Add((sortedSetDict[member], member)); } @@ -568,6 +576,12 @@ private void SortedSetRange(ref ObjectInput input, ref SpanByteAndMemory output) // calculate number of elements var n = maxIndex - minIndex + 1; var iterator = options.Reverse ? sortedSet.Reverse() : sortedSet; + + if (expirationTimes is not null) + { + iterator = iterator.Where(x => !IsExpired(x.Element)); + } + iterator = iterator.Skip(minIndex).Take(n); WriteSortedSetResult(options.WithScores, n, respProtocolVersion, iterator, ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); @@ -642,6 +656,8 @@ void WriteSortedSetResult(bool withScores, int count, int respProtocolVersion, I private void SortedSetRemoveRangeByRank(ref ObjectInput input, ref SpanByteAndMemory output) { + DeleteExpiredItems(); + // ZREMRANGEBYRANK key start stop var isMemory = false; MemoryHandle ptrHandle = default; @@ -683,6 +699,7 @@ private void SortedSetRemoveRangeByRank(ref ObjectInput input, ref SpanByteAndMe this.UpdateSize(item.Item2, false); } + TryRemoveExpiration(item.Item2); } // Write the number of elements @@ -701,6 +718,8 @@ private void SortedSetRemoveRangeByRank(ref ObjectInput input, ref SpanByteAndMe private void SortedSetRemoveRangeByScore(ref ObjectInput input, ref SpanByteAndMemory output) { + DeleteExpiredItems(); + // ZREMRANGEBYSCORE key min max var isMemory = false; MemoryHandle ptrHandle = default; @@ -748,9 +767,10 @@ private void SortedSetRandomMember(ref ObjectInput input, ref SpanByteAndMemory var withScores = (input.arg1 & 1) == 1; var includedCount = ((input.arg1 >> 1) & 1) == 1; var seed = input.arg2; + var sortedSetCount = Count(); - if (count > 0 && count > sortedSet.Count) - count = sortedSet.Count; + if (count > 0 && count > sortedSetCount) + count = sortedSetCount; var isMemory = false; MemoryHandle ptrHandle = default; @@ -770,11 +790,11 @@ private void SortedSetRandomMember(ref ObjectInput input, ref SpanByteAndMemory ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); } - var indexes = RandomUtils.PickKRandomIndexes(sortedSetDict.Count, Math.Abs(count), seed, count > 0); + var indexes = RandomUtils.PickKRandomIndexes(sortedSetCount, Math.Abs(count), seed, count > 0); foreach (var item in indexes) { - var (element, score) = sortedSetDict.ElementAt(item); + var (element, score) = ElementAt(item); while (!RespWriteUtils.TryWriteBulkString(element, ref curr, end)) ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); @@ -812,7 +832,14 @@ private void SortedSetRemoveOrCountRangeByLex(ref ObjectInput input, byte* outpu var minParamBytes = input.parseState.GetArgSliceByRef(0).ReadOnlySpan; var maxParamBytes = input.parseState.GetArgSliceByRef(1).ReadOnlySpan; - var rem = GetElementsInRangeByLex(minParamBytes, maxParamBytes, false, false, op != SortedSetOperation.ZLEXCOUNT, out int errorCode); + var isRemove = op == SortedSetOperation.ZREMRANGEBYLEX; + + if (isRemove) + { + DeleteExpiredItems(); + } + + var rem = GetElementsInRangeByLex(minParamBytes, maxParamBytes, false, false, isRemove, out int errorCode); _output->result1 = errorCode; if (errorCode == 0) @@ -843,7 +870,7 @@ private void SortedSetRank(ref ObjectInput input, ref SpanByteAndMemory output, { var member = input.parseState.GetArgSliceByRef(0).SpanByte.ToByteArray(); - if (!sortedSetDict.TryGetValue(member, out var score)) + if (!TryGetScore(member, out var score)) { while (!RespWriteUtils.TryWriteNull(ref curr, end)) ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); @@ -853,13 +880,18 @@ private void SortedSetRank(ref ObjectInput input, ref SpanByteAndMemory output, var rank = 0; foreach (var item in sortedSet) { + if (IsExpired(item.Element)) + { + continue; + } + if (item.Item2.SequenceEqual(member)) break; rank++; } if (!ascending) - rank = sortedSet.Count - rank - 1; + rank = Count() - rank - 1; if (withScore) { @@ -897,12 +929,15 @@ private void SortedSetRank(ref ObjectInput input, ref SpanByteAndMemory output, /// A tuple containing the score and the element as a byte array. public (double Score, byte[] Element) PopMinOrMax(bool popMaxScoreElement = false) { + DeleteExpiredItems(); + if (sortedSet.Count == 0) return default; var element = popMaxScoreElement ? sortedSet.Max : sortedSet.Min; sortedSet.Remove(element); sortedSetDict.Remove(element.Element); + TryRemoveExpiration(element.Element); this.UpdateSize(element.Element, false); return element; @@ -916,6 +951,8 @@ private void SortedSetRank(ref ObjectInput input, ref SpanByteAndMemory output, /// private void SortedSetPopMinOrMaxCount(ref ObjectInput input, ref SpanByteAndMemory output, SortedSetOperation op) { + DeleteExpiredItems(); + var count = input.arg1; var countDone = 0; @@ -941,6 +978,7 @@ private void SortedSetPopMinOrMaxCount(ref ObjectInput input, ref SpanByteAndMem var max = op == SortedSetOperation.ZPOPMAX ? sortedSet.Max : sortedSet.Min; sortedSet.Remove(max); sortedSetDict.Remove(max.Element); + TryRemoveExpiration(max.Element); this.UpdateSize(max.Element, false); @@ -966,6 +1004,150 @@ private void SortedSetPopMinOrMaxCount(ref ObjectInput input, ref SpanByteAndMem } } + + private void SortedSetPersist(ref ObjectInput input, ref SpanByteAndMemory output) + { + var isMemory = false; + MemoryHandle ptrHandle = default; + var ptr = output.SpanByte.ToPointer(); + + var curr = ptr; + var end = curr + output.Length; + + ObjectOutputHeader _output = default; + try + { + DeleteExpiredItems(); + + var numFields = input.parseState.Count; + while (!RespWriteUtils.TryWriteArrayLength(numFields, ref curr, end)) + ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); + + foreach (var item in input.parseState.Parameters) + { + var result = Persist(item.ToArray()); + while (!RespWriteUtils.TryWriteInt32(result, ref curr, end)) + ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); + _output.result1++; + } + } + finally + { + while (!RespWriteUtils.TryWriteDirect(ref _output, ref curr, end)) + ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); + + if (isMemory) ptrHandle.Dispose(); + output.Length = (int)(curr - ptr); + } + } + + private void SortedSetTimeToLive(ref ObjectInput input, ref SpanByteAndMemory output) + { + var isMemory = false; + MemoryHandle ptrHandle = default; + var ptr = output.SpanByte.ToPointer(); + + var curr = ptr; + var end = curr + output.Length; + + ObjectOutputHeader _output = default; + try + { + DeleteExpiredItems(); + + var isMilliseconds = input.arg1 == 1; + var isTimestamp = input.arg2 == 1; + var numFields = input.parseState.Count; + while (!RespWriteUtils.TryWriteArrayLength(numFields, ref curr, end)) + ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); + + foreach (var item in input.parseState.Parameters) + { + var result = GetExpiration(item.ToArray()); + + if (result >= 0) + { + if (isTimestamp && isMilliseconds) + { + result = ConvertUtils.UnixTimeInMillisecondsFromTicks(result); + } + else if (isTimestamp && !isMilliseconds) + { + result = ConvertUtils.UnixTimeInSecondsFromTicks(result); + } + else if (!isTimestamp && isMilliseconds) + { + result = ConvertUtils.MillisecondsFromDiffUtcNowTicks(result); + } + else if (!isTimestamp && !isMilliseconds) + { + result = ConvertUtils.SecondsFromDiffUtcNowTicks(result); + } + } + + while (!RespWriteUtils.TryWriteInt64(result, ref curr, end)) + ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); + _output.result1++; + } + } + finally + { + while (!RespWriteUtils.TryWriteDirect(ref _output, ref curr, end)) + ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); + + if (isMemory) ptrHandle.Dispose(); + output.Length = (int)(curr - ptr); + } + } + + private void SortedSetExpire(ref ObjectInput input, ref SpanByteAndMemory output) + { + var isMemory = false; + MemoryHandle ptrHandle = default; + var ptr = output.SpanByte.ToPointer(); + + var curr = ptr; + var end = curr + output.Length; + + ObjectOutputHeader _output = default; + try + { + DeleteExpiredItems(); + + var expireOption = (ExpireOption)input.arg1; + var expiration = input.parseState.GetLong(0); + var numFields = input.parseState.Count - 1; + while (!RespWriteUtils.TryWriteArrayLength(numFields, ref curr, end)) + ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); + + foreach (var item in input.parseState.Parameters.Slice(1)) + { + var result = SetExpiration(item.ToArray(), expiration, expireOption); + while (!RespWriteUtils.TryWriteInt32(result, ref curr, end)) + ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); + _output.result1++; + } + } + finally + { + while (!RespWriteUtils.TryWriteDirect(ref _output, ref curr, end)) + ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); + + if (isMemory) ptrHandle.Dispose(); + output.Length = (int)(curr - ptr); + } + } + + private void SortedSetCollect(ref ObjectInput input, byte* output) + { + var _output = (ObjectOutputHeader*)output; + *_output = default; + + DeleteExpiredItems(); + + _output->result1 = 1; + } + #region CommonMethods /// @@ -1012,6 +1194,11 @@ private void SortedSetPopMinOrMaxCount(ref ObjectInput input, ref SpanByteAndMem // using ToList method so we avoid the Invalid operation ex. when removing foreach (var item in iterator.ToList()) { + if (IsExpired(item.Element)) + { + continue; + } + var inRange = new ReadOnlySpan(item.Item2).SequenceCompareTo(minValueChars); if (inRange < 0 || (inRange == 0 && minValueExclusive)) continue; @@ -1026,6 +1213,7 @@ private void SortedSetPopMinOrMaxCount(ref ObjectInput input, ref SpanByteAndMem { sortedSetDict.Remove(item.Item2); sortedSet.Remove((_key, item.Item2)); + TryRemoveExpiration(item.Element); this.UpdateSize(item.Item2, false); } @@ -1082,6 +1270,7 @@ private void SortedSetPopMinOrMaxCount(ref ObjectInput input, ref SpanByteAndMem foreach (var item in sortedSet.GetViewBetween((minValue, null), sortedSet.Max)) { + if (IsExpired(item.Element)) continue; if (item.Item1 > maxValue || (maxExclusive && item.Item1 == maxValue)) break; if (minExclusive && item.Item1 == minValue) continue; scoredElements.Add(item); @@ -1103,6 +1292,7 @@ private void SortedSetPopMinOrMaxCount(ref ObjectInput input, ref SpanByteAndMem { sortedSetDict.Remove(item.Item2); sortedSet.Remove((_key, item.Item2)); + TryRemoveExpiration(item.Item2); this.UpdateSize(item.Item2, false); } diff --git a/libs/server/Objects/Types/GarnetObject.cs b/libs/server/Objects/Types/GarnetObject.cs index 7474d547e7..85167c197b 100644 --- a/libs/server/Objects/Types/GarnetObject.cs +++ b/libs/server/Objects/Types/GarnetObject.cs @@ -43,6 +43,8 @@ internal static bool NeedToCreate(RespInputHeader header) SortedSetOperation.ZREMRANGEBYLEX => false, SortedSetOperation.ZREMRANGEBYSCORE => false, SortedSetOperation.ZREMRANGEBYRANK => false, + SortedSetOperation.ZEXPIRE => false, + SortedSetOperation.ZCOLLECT => false, _ => true, }, GarnetObjectType.List => header.ListOp switch diff --git a/libs/server/Resp/AdminCommands.cs b/libs/server/Resp/AdminCommands.cs index 26b791fc24..d5328dc634 100644 --- a/libs/server/Resp/AdminCommands.cs +++ b/libs/server/Resp/AdminCommands.cs @@ -54,6 +54,7 @@ RespCommand.MIGRATE or RespCommand.COMMITAOF => NetworkCOMMITAOF(), RespCommand.FORCEGC => NetworkFORCEGC(), RespCommand.HCOLLECT => NetworkHCOLLECT(ref storageApi), + RespCommand.ZCOLLECT => NetworkZCOLLECT(ref storageApi), RespCommand.MONITOR => NetworkMonitor(), RespCommand.ACL_DELUSER => NetworkAclDelUser(), RespCommand.ACL_LIST => NetworkAclList(), @@ -634,6 +635,36 @@ private bool NetworkHCOLLECT(ref TGarnetApi storageApi) return true; } + private bool NetworZHCOLLECT(ref TGarnetApi storageApi) + where TGarnetApi : IGarnetApi + { + if (parseState.Count < 1) + { + return AbortWithWrongNumberOfArguments(nameof(RespCommand.ZCOLLECT)); + } + + var keys = parseState.Parameters; + + var header = new RespInputHeader(GarnetObjectType.SortedSet) { SortedSetOp = SortedSetOperation.ZCOLLECT }; + var input = new ObjectInput(header); + + var status = storageApi.SortedSetCollect(keys, ref input); + + switch (status) + { + case GarnetStatus.OK: + while (!RespWriteUtils.TryWriteDirect(CmdStrings.RESP_OK, ref dcurr, dend)) + SendAndReset(); + break; + default: + while (!RespWriteUtils.TryWriteError(CmdStrings.RESP_ERR_ZCOLLECT_ALREADY_IN_PROGRESS, ref dcurr, dend)) + SendAndReset(); + break; + } + + return true; + } + private bool NetworkProcessClusterCommand(RespCommand command) { if (clusterSession == null) diff --git a/libs/server/Resp/BasicCommands.cs b/libs/server/Resp/BasicCommands.cs index 06a540d057..d8b253485e 100644 --- a/libs/server/Resp/BasicCommands.cs +++ b/libs/server/Resp/BasicCommands.cs @@ -3,7 +3,6 @@ using System; using System.Buffers; -using System.Collections.Generic; using System.Diagnostics; using System.Text; using System.Threading.Tasks; diff --git a/libs/server/Resp/CmdStrings.cs b/libs/server/Resp/CmdStrings.cs index bdafb73705..f30ff42300 100644 --- a/libs/server/Resp/CmdStrings.cs +++ b/libs/server/Resp/CmdStrings.cs @@ -113,6 +113,7 @@ static partial class CmdStrings public static ReadOnlySpan maxlen => "maxlen"u8; public static ReadOnlySpan PUBSUB => "PUBSUB"u8; public static ReadOnlySpan HCOLLECT => "HCOLLECT"u8; + public static ReadOnlySpan ZCOLLECT => "ZCOLLECT"u8; public static ReadOnlySpan CHANNELS => "CHANNELS"u8; public static ReadOnlySpan NUMPAT => "NUMPAT"u8; public static ReadOnlySpan NUMSUB => "NUMSUB"u8; @@ -141,6 +142,7 @@ static partial class CmdStrings public static ReadOnlySpan GETIFNOTMATCH => "GETIFNOTMATCH"u8; public static ReadOnlySpan SETIFMATCH => "SETIFMATCH"u8; public static ReadOnlySpan FIELDS => "FIELDS"u8; + public static ReadOnlySpan MEMBERS => "MEMBERS"u8; public static ReadOnlySpan TIMEOUT => "TIMEOUT"u8; public static ReadOnlySpan ERROR => "ERROR"u8; @@ -244,6 +246,7 @@ static partial class CmdStrings public static ReadOnlySpan RESP_ERR_LENGTH_AND_INDEXES => "If you want both the length and indexes, please just use IDX."u8; public static ReadOnlySpan RESP_ERR_INVALID_EXPIRE_TIME => "ERR invalid expire time, must be >= 0"u8; public static ReadOnlySpan RESP_ERR_HCOLLECT_ALREADY_IN_PROGRESS => "ERR HCOLLECT scan already in progress"u8; + public static ReadOnlySpan RESP_ERR_ZCOLLECT_ALREADY_IN_PROGRESS => "ERR ZCOLLECT scan already in progress"u8; public static ReadOnlySpan RESP_INVALID_COMMAND_SPECIFIED => "Invalid command specified"u8; public static ReadOnlySpan RESP_COMMAND_HAS_NO_KEY_ARGS => "The command has no key arguments"u8; public static ReadOnlySpan RESP_ERR_INVALID_CLIENT_UNBLOCK_REASON => "ERR CLIENT UNBLOCK reason should be TIMEOUT or ERROR"u8; diff --git a/libs/server/Resp/Objects/SortedSetCommands.cs b/libs/server/Resp/Objects/SortedSetCommands.cs index e04c353fa4..48b2773164 100644 --- a/libs/server/Resp/Objects/SortedSetCommands.cs +++ b/libs/server/Resp/Objects/SortedSetCommands.cs @@ -1682,5 +1682,265 @@ private unsafe bool SortedSetBlockingMPop() return true; } + + /// + /// Sets an expiration time for a member in the SortedSet stored at key. + /// + /// + /// + /// + /// + private unsafe bool SortedSetExpire(RespCommand command, ref TGarnetApi storageApi) + where TGarnetApi : IGarnetApi + { + if (storeWrapper.itemBroker == null) + throw new GarnetException("Object store is disabled"); + + if (parseState.Count <= 4) + { + return AbortWithWrongNumberOfArguments(command.ToString()); + } + + var key = parseState.GetArgSliceByRef(0); + + long expireAt = 0; + var isMilliseconds = false; + if (!parseState.TryGetLong(1, out expireAt)) + { + return AbortWithErrorMessage(CmdStrings.RESP_ERR_GENERIC_VALUE_IS_NOT_INTEGER); + } + + if (expireAt < 0) + { + return AbortWithErrorMessage(CmdStrings.RESP_ERR_INVALID_EXPIRE_TIME); + } + + switch (command) + { + case RespCommand.ZEXPIRE: + expireAt = DateTimeOffset.UtcNow.ToUnixTimeSeconds() + expireAt; + isMilliseconds = false; + break; + case RespCommand.ZPEXPIRE: + expireAt = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds() + expireAt; + isMilliseconds = true; + break; + case RespCommand.ZPEXPIREAT: + isMilliseconds = true; + break; + default: // RespCommand.ZEXPIREAT + break; + } + + var currIdx = 2; + if (parseState.TryGetExpireOption(currIdx, out var expireOption)) + { + currIdx++; // If expire option is present, move to next argument else continue with the current argument + } + + var fieldOption = parseState.GetArgSliceByRef(currIdx++); + if (!fieldOption.ReadOnlySpan.EqualsUpperCaseSpanIgnoringCase(CmdStrings.MEMBERS)) + { + return AbortWithErrorMessage(Encoding.ASCII.GetBytes(string.Format(CmdStrings.GenericErrMandatoryMissing, "MEMBERS"))); + } + + if (!parseState.TryGetInt(currIdx++, out var numMembers)) + { + return AbortWithErrorMessage(Encoding.ASCII.GetBytes(string.Format(CmdStrings.GenericParamShouldBeGreaterThanZero, "numMembers"))); + } + + if (parseState.Count != currIdx + numMembers) + { + return AbortWithErrorMessage(Encoding.ASCII.GetBytes(string.Format(CmdStrings.GenericErrMustMatchNoOfArgs, "numMembers"))); + } + + var membersParseState = parseState.Slice(currIdx, numMembers); + + var header = new RespInputHeader(GarnetObjectType.SortedSet) { SortedSetOp = SortedSetOperation.ZEXPIRE }; + var input = new ObjectInput(header, ref membersParseState); + + var outputFooter = new GarnetObjectStoreOutput { SpanByteAndMemory = new SpanByteAndMemory(dcurr, (int)(dend - dcurr)) }; + + var status = storageApi.SortedSetExpire(key, expireAt, isMilliseconds, expireOption, ref input, ref outputFooter); + + switch (status) + { + case GarnetStatus.WRONGTYPE: + while (!RespWriteUtils.TryWriteError(CmdStrings.RESP_ERR_WRONG_TYPE, ref dcurr, dend)) + SendAndReset(); + break; + case GarnetStatus.NOTFOUND: + while (!RespWriteUtils.TryWriteArrayLength(numMembers, ref dcurr, dend)) + SendAndReset(); + for (var i = 0; i < numMembers; i++) + { + while (!RespWriteUtils.TryWriteInt32(-2, ref dcurr, dend)) + SendAndReset(); + } + break; + default: + ProcessOutputWithHeader(outputFooter.SpanByteAndMemory); + break; + } + + return true; + } + + /// + /// Returns the time to live (TTL) for the specified members in the SortedSet stored at the given key. + /// + /// The type of the storage API. + /// The RESP command indicating the type of TTL operation. + /// The storage API instance to interact with the underlying storage. + /// True if the operation was successful; otherwise, false. + /// Thrown when the object store is disabled. + private unsafe bool SortedSetTimeToLive(RespCommand command, ref TGarnetApi storageApi) + where TGarnetApi : IGarnetApi + { + if (storeWrapper.itemBroker == null) + throw new GarnetException("Object store is disabled"); + + if (parseState.Count <= 3) + { + return AbortWithWrongNumberOfArguments(command.ToString()); + } + + var key = parseState.GetArgSliceByRef(0); + + var fieldOption = parseState.GetArgSliceByRef(1); + if (!fieldOption.ReadOnlySpan.EqualsUpperCaseSpanIgnoringCase(CmdStrings.MEMBERS)) + { + return AbortWithErrorMessage(Encoding.ASCII.GetBytes(string.Format(CmdStrings.GenericErrMandatoryMissing, "MEMBERS"))); + } + + if (!parseState.TryGetInt(2, out var numMembers)) + { + return AbortWithErrorMessage(Encoding.ASCII.GetBytes(string.Format(CmdStrings.GenericParamShouldBeGreaterThanZero, "numMembers"))); + } + + if (parseState.Count != 3 + numMembers) + { + return AbortWithErrorMessage(Encoding.ASCII.GetBytes(string.Format(CmdStrings.GenericErrMustMatchNoOfArgs, "numMembers"))); + } + + var isMilliseconds = false; + var isTimestamp = false; + switch (command) + { + case RespCommand.ZPTTL: + isMilliseconds = true; + isTimestamp = false; + break; + case RespCommand.ZEXPIRETIME: + isMilliseconds = false; + isTimestamp = true; + break; + case RespCommand.ZPEXPIRETIME: + isMilliseconds = true; + isTimestamp = true; + break; + default: // RespCommand.ZTTL + break; + } + + var membersParseState = parseState.Slice(3, numMembers); + + var header = new RespInputHeader(GarnetObjectType.SortedSet) { SortedSetOp = SortedSetOperation.ZTTL }; + var input = new ObjectInput(header, ref membersParseState); + + var outputFooter = new GarnetObjectStoreOutput { SpanByteAndMemory = new SpanByteAndMemory(dcurr, (int)(dend - dcurr)) }; + + var status = storageApi.SortedSetTimeToLive(key, isMilliseconds, isTimestamp, ref input, ref outputFooter); + + switch (status) + { + case GarnetStatus.WRONGTYPE: + while (!RespWriteUtils.TryWriteError(CmdStrings.RESP_ERR_WRONG_TYPE, ref dcurr, dend)) + SendAndReset(); + break; + case GarnetStatus.NOTFOUND: + while (!RespWriteUtils.TryWriteArrayLength(numMembers, ref dcurr, dend)) + SendAndReset(); + for (var i = 0; i < numMembers; i++) + { + while (!RespWriteUtils.TryWriteInt32(-2, ref dcurr, dend)) + SendAndReset(); + } + break; + default: + ProcessOutputWithHeader(outputFooter.SpanByteAndMemory); + break; + } + + return true; + } + + /// + /// Removes the expiration time from the specified members in the sorted set stored at the given key. + /// + /// The type of the storage API. + /// The storage API instance to interact with the underlying storage. + /// True if the operation was successful; otherwise, false. + /// Thrown when the object store is disabled. + private unsafe bool SortedSetPersist(ref TGarnetApi storageApi) + where TGarnetApi : IGarnetApi + { + if (storeWrapper.itemBroker == null) + throw new GarnetException("Object store is disabled"); + + if (parseState.Count <= 3) + { + return AbortWithWrongNumberOfArguments(nameof(RespCommand.ZPERSIST)); + } + + var key = parseState.GetArgSliceByRef(0); + + var fieldOption = parseState.GetArgSliceByRef(1); + if (!fieldOption.ReadOnlySpan.EqualsUpperCaseSpanIgnoringCase(CmdStrings.MEMBERS)) + { + return AbortWithErrorMessage(Encoding.ASCII.GetBytes(string.Format(CmdStrings.GenericErrMandatoryMissing, "MEMBERS"))); + } + + if (!parseState.TryGetInt(2, out var numMembers)) + { + return AbortWithErrorMessage(Encoding.ASCII.GetBytes(string.Format(CmdStrings.GenericParamShouldBeGreaterThanZero, "numMembers"))); + } + + if (parseState.Count != 3 + numMembers) + { + return AbortWithErrorMessage(Encoding.ASCII.GetBytes(string.Format(CmdStrings.GenericErrMustMatchNoOfArgs, "numMembers"))); + } + + var membersParseState = parseState.Slice(3, numMembers); + + var header = new RespInputHeader(GarnetObjectType.SortedSet) { SortedSetOp = SortedSetOperation.ZPERSIST }; + var input = new ObjectInput(header, ref membersParseState); + + var outputFooter = new GarnetObjectStoreOutput { SpanByteAndMemory = new SpanByteAndMemory(dcurr, (int)(dend - dcurr)) }; + + var status = storageApi.SortedSetPersist(key, ref input, ref outputFooter); + + switch (status) + { + case GarnetStatus.WRONGTYPE: + while (!RespWriteUtils.TryWriteError(CmdStrings.RESP_ERR_WRONG_TYPE, ref dcurr, dend)) + SendAndReset(); + break; + case GarnetStatus.NOTFOUND: + while (!RespWriteUtils.TryWriteArrayLength(numMembers, ref dcurr, dend)) + SendAndReset(); + for (var i = 0; i < numMembers; i++) + { + while (!RespWriteUtils.TryWriteInt32(-2, ref dcurr, dend)) + SendAndReset(); + } + break; + default: + ProcessOutputWithHeader(outputFooter.SpanByteAndMemory); + break; + } + + return true; + } } } \ No newline at end of file diff --git a/libs/server/Resp/Parser/RespCommand.cs b/libs/server/Resp/Parser/RespCommand.cs index dbb14372ce..0dd80a70f4 100644 --- a/libs/server/Resp/Parser/RespCommand.cs +++ b/libs/server/Resp/Parser/RespCommand.cs @@ -97,6 +97,10 @@ public enum RespCommand : ushort ZREVRANGEBYLEX, ZREVRANGEBYSCORE, ZREVRANK, + ZTTL, + ZPTTL, + ZEXPIRETIME, + ZPEXPIRETIME, ZSCAN, ZSCORE, // Note: Last read command should immediately precede FirstWriteCommand ZUNION, @@ -183,7 +187,13 @@ public enum RespCommand : ushort SUNIONSTORE, UNLINK, ZADD, + ZCOLLECT, ZDIFFSTORE, + ZEXPIRE, + ZPEXPIRE, + ZEXPIREAT, + ZPEXPIREAT, + ZPERSIST, ZINCRBY, ZMPOP, ZINTERSTORE, @@ -842,6 +852,10 @@ private RespCommand FastParseArrayCommand(ref int count, ref ReadOnlySpan { return RespCommand.HTTL; } + else if (*(ulong*)(ptr + 2) == MemoryMarshal.Read("\r\nZTTL\r\n"u8)) + { + return RespCommand.ZTTL; + } break; case 'K': @@ -1020,6 +1034,10 @@ private RespCommand FastParseArrayCommand(ref int count, ref ReadOnlySpan { return RespCommand.HPTTL; } + else if (*(ulong*)(ptr + 3) == MemoryMarshal.Read("\nZPTTL\r\n"u8)) + { + return RespCommand.ZPTTL; + } break; case 'L': @@ -1319,6 +1337,10 @@ private RespCommand FastParseArrayCommand(ref int count, ref ReadOnlySpan { return RespCommand.HEXPIRE; } + else if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("ZEXPIRE\r"u8) && *(byte*)(ptr + 12) == '\n') + { + return RespCommand.ZEXPIRE; + } else if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("HINCRBY\r"u8) && *(byte*)(ptr + 12) == '\n') { return RespCommand.HINCRBY; @@ -1411,6 +1433,14 @@ private RespCommand FastParseArrayCommand(ref int count, ref ReadOnlySpan { return RespCommand.HPERSIST; } + else if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("ZPEXPIRE"u8) && *(ushort*)(ptr + 12) == MemoryMarshal.Read("\r\n"u8)) + { + return RespCommand.ZPEXPIRE; + } + else if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("ZPERSIST"u8) && *(ushort*)(ptr + 12) == MemoryMarshal.Read("\r\n"u8)) + { + return RespCommand.ZPERSIST; + } else if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("BZPOPMAX"u8) && *(ushort*)(ptr + 12) == MemoryMarshal.Read("\r\n"u8)) { return RespCommand.BZPOPMAX; @@ -1453,6 +1483,10 @@ private RespCommand FastParseArrayCommand(ref int count, ref ReadOnlySpan { return RespCommand.HEXPIREAT; } + else if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("ZEXPIREA"u8) && *(uint*)(ptr + 11) == MemoryMarshal.Read("AT\r\n"u8)) + { + return RespCommand.ZEXPIREAT; + } break; case 10: if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("SSUBSCRI"u8) && *(uint*)(ptr + 11) == MemoryMarshal.Read("BE\r\n"u8)) @@ -1528,6 +1562,10 @@ private RespCommand FastParseArrayCommand(ref int count, ref ReadOnlySpan { return RespCommand.HPEXPIREAT; } + else if (*(ulong*)(ptr + 1) == MemoryMarshal.Read("10\r\nZPEX"u8) && *(uint*)(ptr + 9) == MemoryMarshal.Read("PIREAT\r\n"u8)) + { + return RespCommand.ZPEXPIREAT; + } break; case 11: if (*(ulong*)(ptr + 2) == MemoryMarshal.Read("1\r\nUNSUB"u8) && *(ulong*)(ptr + 10) == MemoryMarshal.Read("SCRIBE\r\n"u8)) @@ -1574,9 +1612,9 @@ private RespCommand FastParseArrayCommand(ref int count, ref ReadOnlySpan { return RespCommand.ZUNIONSTORE; } - else if (*(ulong*)(ptr + 2) == MemoryMarshal.Read("1\r\nHEXPI"u8) && *(ulong*)(ptr + 10) == MemoryMarshal.Read("RETIME\r\n"u8)) + else if (*(ulong*)(ptr + 2) == MemoryMarshal.Read("1\r\nZEXPI"u8) && *(ulong*)(ptr + 10) == MemoryMarshal.Read("RETIME\r\n"u8)) { - return RespCommand.HEXPIRETIME; + return RespCommand.ZEXPIRETIME; } break; @@ -1593,6 +1631,10 @@ private RespCommand FastParseArrayCommand(ref int count, ref ReadOnlySpan { return RespCommand.HPEXPIRETIME; } + else if (*(ulong*)(ptr + 3) == MemoryMarshal.Read("\r\nZPEXPI"u8) && *(ulong*)(ptr + 11) == MemoryMarshal.Read("RETIME\r\n"u8)) + { + return RespCommand.ZPEXPIRETIME; + } break; case 13: @@ -2237,6 +2279,10 @@ private RespCommand SlowParseCommand(ref int count, ref ReadOnlySpan speci { return RespCommand.HCOLLECT; } + else if (command.SequenceEqual(CmdStrings.ZCOLLECT)) + { + return RespCommand.ZCOLLECT; + } else { // Custom commands should have never been set when we reach this point diff --git a/libs/server/Resp/RespServerSession.cs b/libs/server/Resp/RespServerSession.cs index 940d147be2..2fd9b71453 100644 --- a/libs/server/Resp/RespServerSession.cs +++ b/libs/server/Resp/RespServerSession.cs @@ -660,6 +660,15 @@ private bool ProcessArrayCommands(RespCommand cmd, ref TGarnetApi st RespCommand.ZINTERSTORE => SortedSetIntersectStore(ref storageApi), RespCommand.ZUNION => SortedSetUnion(ref storageApi), RespCommand.ZUNIONSTORE => SortedSetUnionStore(ref storageApi), + RespCommand.ZEXPIRE => SortedSetExpire(cmd, ref storageApi), + RespCommand.ZPEXPIRE => SortedSetExpire(cmd, ref storageApi), + RespCommand.ZEXPIREAT => SortedSetExpire(cmd, ref storageApi), + RespCommand.ZPEXPIREAT => SortedSetExpire(cmd, ref storageApi), + RespCommand.ZTTL => SortedSetTimeToLive(cmd, ref storageApi), + RespCommand.ZPTTL => SortedSetTimeToLive(cmd, ref storageApi), + RespCommand.ZEXPIRETIME => SortedSetTimeToLive(cmd, ref storageApi), + RespCommand.ZPEXPIRETIME => SortedSetTimeToLive(cmd, ref storageApi), + RespCommand.ZPERSIST => SortedSetPersist(ref storageApi), //SortedSet for Geo Commands RespCommand.GEOADD => GeoAdd(ref storageApi), RespCommand.GEOHASH => GeoCommands(cmd, ref storageApi), diff --git a/libs/server/Servers/GarnetServerOptions.cs b/libs/server/Servers/GarnetServerOptions.cs index d248747d1e..8049626eec 100644 --- a/libs/server/Servers/GarnetServerOptions.cs +++ b/libs/server/Servers/GarnetServerOptions.cs @@ -141,6 +141,11 @@ public class GarnetServerOptions : ServerOptions /// public int HashCollectFrequencySecs = 0; + /// + /// Hash collection frequency in seconds. 0 = disabled. Sorted Set collect is used to delete expired members from Sorted Set without waiting for a write operation. + /// + public int SortedSetCollectFrequencySecs = 0; + /// /// Hybrid log compaction type. /// None - no compaction. diff --git a/libs/server/Storage/Session/ObjectStore/SortedSetOps.cs b/libs/server/Storage/Session/ObjectStore/SortedSetOps.cs index d3f90cb840..1ea7e32a61 100644 --- a/libs/server/Storage/Session/ObjectStore/SortedSetOps.cs +++ b/libs/server/Storage/Session/ObjectStore/SortedSetOps.cs @@ -16,6 +16,7 @@ namespace Garnet.server sealed partial class StorageSession : IDisposable { + private SingleWriterMultiReaderLock _zcollectTaskLock; /// /// Adds the specified member and score to the sorted set stored at key. @@ -1498,5 +1499,127 @@ private GarnetStatus SortedSetIntersection(ReadOnlySpan + /// Sets the expiration time for the specified key. + /// + /// The type of the object context. + /// The key for which to set the expiration time. + /// The expiration time in ticks. + /// Indicates whether the expiration time is in milliseconds. + /// The expiration option to use. + /// The input object containing the operation details. + /// The output footer object to store the result. + /// The object context for the operation. + /// The status of the operation. + public GarnetStatus SortedSetExpire(ArgSlice key, long expireAt, bool isMilliseconds, ExpireOption expireOption, ref ObjectInput input, ref GarnetObjectStoreOutput outputFooter, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext + { + var expireAtUtc = isMilliseconds ? ConvertUtils.UnixTimestampInMillisecondsToTicks(expireAt) : ConvertUtils.UnixTimestampInSecondsToTicks(expireAt); + var expiryLength = NumUtils.CountDigits(expireAtUtc); + var expirySlice = scratchBufferManager.CreateArgSlice(expiryLength); + var expirySpan = expirySlice.Span; + NumUtils.WriteInt64(expireAtUtc, expirySpan); + + parseState.Initialize(1 + input.parseState.Count); + parseState.SetArgument(0, expirySlice); + parseState.SetArguments(1, input.parseState.Parameters); + + var innerInput = new ObjectInput(input.header, ref parseState, startIdx: 0, arg1: (int)expireOption); + + return RMWObjectStoreOperationWithOutput(key.ToArray(), ref innerInput, ref objectContext, ref outputFooter); + } + + /// + /// Returns the time-to-live (TTL) of a SortedSet member. + /// + /// The type of the object context. + /// The key of the hash. + /// Indicates whether the TTL is in milliseconds. + /// Indicates whether the TTL is a timestamp. + /// The input object containing the operation details. + /// The output footer object to store the result. + /// The object context for the operation. + /// The status of the operation. + public GarnetStatus SortedSetTimeToLive(ArgSlice key, bool isMilliseconds, bool isTimestamp, ref ObjectInput input, ref GarnetObjectStoreOutput outputFooter, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext + { + var innerInput = new ObjectInput(input.header, ref input.parseState, arg1: isMilliseconds ? 1 : 0, arg2: isTimestamp ? 1 : 0); + + return ReadObjectStoreOperationWithOutput(key.ToArray(), ref innerInput, ref objectContext, ref outputFooter); + } + + /// + /// Removes the expiration time from a SortedSet member, making it persistent. + /// + /// The type of the object context. + /// The key of the SortedSet. + /// The input object containing the operation details. + /// The output footer object to store the result. + /// The object context for the operation. + /// The status of the operation. + public GarnetStatus SortedSetPersist(ArgSlice key, ref ObjectInput input, ref GarnetObjectStoreOutput outputFooter, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext + => RMWObjectStoreOperationWithOutput(key.ToArray(), ref input, ref objectContext, ref outputFooter); + + /// + /// Collects SortedSet keys and performs a specified operation on them. + /// + /// The type of the object context. + /// The keys to collect. + /// The input object containing the operation details. + /// The object context for the operation. + /// The status of the operation. + /// + /// If the first key is "*", all SortedSet keys are scanned in batches and the operation is performed on each key. + /// Otherwise, the operation is performed on the specified keys. + /// + public GarnetStatus SortedSetCollect(ReadOnlySpan keys, ref ObjectInput input, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext + { + if (!_zcollectTaskLock.TryWriteLock()) + { + return GarnetStatus.NOTFOUND; + } + + try + { + if (keys[0].ReadOnlySpan.SequenceEqual("*"u8)) + { + long cursor = 0; + long storeCursor = 0; + + // Scan all SortedSet keys in batches + do + { + if (!DbScan(keys[0], true, cursor, out storeCursor, out var hashKeys, 100, CmdStrings.ZSET)) + { + return GarnetStatus.OK; + } + + // Process each SortedSet key + foreach (var hashKey in hashKeys) + { + RMWObjectStoreOperation(hashKey, ref input, out _, ref objectContext); + } + + cursor = storeCursor; + } while (storeCursor != 0); + + return GarnetStatus.OK; + } + + foreach (var key in keys) + { + RMWObjectStoreOperation(key.ToArray(), ref input, out _, ref objectContext); + } + + return GarnetStatus.OK; + } + finally + { + _zcollectTaskLock.WriteUnlock(); + } + } } } \ No newline at end of file diff --git a/libs/server/StoreWrapper.cs b/libs/server/StoreWrapper.cs index 510b5d154b..7f4004cd39 100644 --- a/libs/server/StoreWrapper.cs +++ b/libs/server/StoreWrapper.cs @@ -468,6 +468,49 @@ static void ExecuteHashCollect(ScratchBufferManager scratchBufferManager, Storag } } + async Task SortedSetCollectTask(int sortedSetCollectFrequencySecs, CancellationToken token = default) + { + Debug.Assert(sortedSetCollectFrequencySecs > 0); + try + { + var scratchBufferManager = new ScratchBufferManager(); + using var storageSession = new StorageSession(this, scratchBufferManager, null, null, logger); + + if (objectStore is null) + { + logger?.LogWarning("SortedSetCollectFrequencySecs option is configured but Object store is disabled. Stopping the background sortedSet collect task."); + return; + } + + while (true) + { + if (token.IsCancellationRequested) return; + + ExecuteSortedSetCollect(scratchBufferManager, storageSession); + + await Task.Delay(TimeSpan.FromSeconds(sortedSetCollectFrequencySecs), token); + } + } + catch (TaskCanceledException) when (token.IsCancellationRequested) + { + // Suppress the exception if the task was cancelled because of store wrapper disposal + } + catch (Exception ex) + { + logger?.LogCritical(ex, "Unknown exception received for background sortedSet collect task. SortedSet collect task won't be resumed."); + } + + static void ExecuteSortedSetCollect(ScratchBufferManager scratchBufferManager, StorageSession storageSession) + { + var header = new RespInputHeader(GarnetObjectType.SortedSet) { SortedSetOp = SortedSetOperation.ZCOLLECT }; + var input = new ObjectInput(header); + + ReadOnlySpan key = [ArgSlice.FromPinnedSpan("*"u8)]; + storageSession.SortedSetCollect(key, ref input, ref storageSession.objectStoreBasicContext); + scratchBufferManager.Reset(); + } + } + void DoCompaction() { // Periodic compaction -> no need to compact before checkpointing @@ -627,6 +670,11 @@ internal void Start() Task.Run(async () => await HashCollectTask(serverOptions.HashCollectFrequencySecs, ctsCommit.Token)); } + if (serverOptions.SortedSetCollectFrequencySecs > 0) + { + Task.Run(async () => await SortedSetCollectTask(serverOptions.SortedSetCollectFrequencySecs, ctsCommit.Token)); + } + if (serverOptions.AdjustedIndexMaxCacheLines > 0 || serverOptions.AdjustedObjectStoreIndexMaxCacheLines > 0) { Task.Run(() => IndexAutoGrowTask(ctsCommit.Token)); diff --git a/playground/CommandInfoUpdater/SupportedCommand.cs b/playground/CommandInfoUpdater/SupportedCommand.cs index 33b49862e9..e530abcfc6 100644 --- a/playground/CommandInfoUpdater/SupportedCommand.cs +++ b/playground/CommandInfoUpdater/SupportedCommand.cs @@ -319,6 +319,16 @@ public class SupportedCommand new("ZREVRANK", RespCommand.ZREVRANK), new("ZSCAN", RespCommand.ZSCAN), new("ZSCORE", RespCommand.ZSCORE), + new("ZEXPIRE", RespCommand.HEXPIRE), + new("ZPEXPIRE", RespCommand.HPEXPIRE), + new("ZEXPIREAT", RespCommand.HEXPIREAT), + new("ZPEXPIREAT", RespCommand.HPEXPIREAT), + new("ZTTL", RespCommand.HTTL), + new("ZPTTL", RespCommand.HPTTL), + new("ZEXPIRETIME", RespCommand.HEXPIRETIME), + new("ZPEXPIRETIME", RespCommand.HPEXPIRETIME), + new("ZPERSIST", RespCommand.HPERSIST), + new("ZCOLLECT", RespCommand.HPERSIST), new("ZUNION", RespCommand.ZUNION), new("ZUNIONSTORE", RespCommand.ZUNIONSTORE), new("EVAL", RespCommand.EVAL), diff --git a/test/Garnet.test.cluster/RedirectTests/BaseCommand.cs b/test/Garnet.test.cluster/RedirectTests/BaseCommand.cs index ee2f31bb68..296364f1f1 100644 --- a/test/Garnet.test.cluster/RedirectTests/BaseCommand.cs +++ b/test/Garnet.test.cluster/RedirectTests/BaseCommand.cs @@ -2317,6 +2317,184 @@ public override ArraySegment[] SetupSingleSlotRequest() } } + internal class ZEXPIRE : BaseCommand + { + public override bool IsArrayCommand => false; + public override bool ArrayResponse => true; + public override string Command => nameof(ZEXPIRE); + + public override string[] GetSingleSlotRequest() + { + var ssk = GetSingleSlotKeys; + return [ssk[0], "3", "MEMBERS", "1", "member1"]; + } + + public override string[] GetCrossSlotRequest() => throw new NotImplementedException(); + + public override ArraySegment[] SetupSingleSlotRequest() => throw new NotImplementedException(); + } + + internal class ZPEXPIRE : BaseCommand + { + public override bool IsArrayCommand => false; + public override bool ArrayResponse => true; + public override string Command => nameof(ZPEXPIRE); + + public override string[] GetSingleSlotRequest() + { + var ssk = GetSingleSlotKeys; + return [ssk[0], "3000", "MEMBERS", "1", "member1"]; + } + + public override string[] GetCrossSlotRequest() => throw new NotImplementedException(); + + public override ArraySegment[] SetupSingleSlotRequest() => throw new NotImplementedException(); + } + + internal class ZEXPIREAT : BaseCommand + { + public override bool IsArrayCommand => false; + public override bool ArrayResponse => true; + public override string Command => nameof(ZEXPIREAT); + + public override string[] GetSingleSlotRequest() + { + var timestamp = DateTimeOffset.UtcNow.AddSeconds(3).ToUnixTimeSeconds().ToString(); + var ssk = GetSingleSlotKeys; + return [ssk[0], timestamp, "MEMBERS", "1", "member1"]; + } + + public override string[] GetCrossSlotRequest() => throw new NotImplementedException(); + + public override ArraySegment[] SetupSingleSlotRequest() => throw new NotImplementedException(); + } + + internal class ZPEXPIREAT : BaseCommand + { + public override bool IsArrayCommand => false; + public override bool ArrayResponse => true; + public override string Command => nameof(ZPEXPIREAT); + + public override string[] GetSingleSlotRequest() + { + var timestamp = DateTimeOffset.UtcNow.AddSeconds(3).ToUnixTimeMilliseconds().ToString(); + var ssk = GetSingleSlotKeys; + return [ssk[0], timestamp, "MEMBERS", "1", "member1"]; + } + + public override string[] GetCrossSlotRequest() => throw new NotImplementedException(); + + public override ArraySegment[] SetupSingleSlotRequest() => throw new NotImplementedException(); + } + + internal class ZTTL : BaseCommand + { + public override bool IsArrayCommand => false; + public override bool ArrayResponse => true; + public override string Command => nameof(ZTTL); + + public override string[] GetSingleSlotRequest() + { + var ssk = GetSingleSlotKeys; + return [ssk[0], "MEMBERS", "1", "member1"]; + } + + public override string[] GetCrossSlotRequest() => throw new NotImplementedException(); + + public override ArraySegment[] SetupSingleSlotRequest() => throw new NotImplementedException(); + } + + internal class ZPTTL : BaseCommand + { + public override bool IsArrayCommand => false; + public override bool ArrayResponse => true; + public override string Command => nameof(ZPTTL); + + public override string[] GetSingleSlotRequest() + { + var ssk = GetSingleSlotKeys; + return [ssk[0], "MEMBERS", "1", "member1"]; + } + + public override string[] GetCrossSlotRequest() => throw new NotImplementedException(); + + public override ArraySegment[] SetupSingleSlotRequest() => throw new NotImplementedException(); + } + + internal class ZEXPIRETIME : BaseCommand + { + public override bool IsArrayCommand => false; + public override bool ArrayResponse => true; + public override string Command => nameof(ZEXPIRETIME); + + public override string[] GetSingleSlotRequest() + { + var ssk = GetSingleSlotKeys; + return [ssk[0], "MEMBERS", "1", "member1"]; + } + + public override string[] GetCrossSlotRequest() => throw new NotImplementedException(); + + public override ArraySegment[] SetupSingleSlotRequest() => throw new NotImplementedException(); + } + + internal class ZPEXPIRETIME : BaseCommand + { + public override bool IsArrayCommand => false; + public override bool ArrayResponse => true; + public override string Command => nameof(ZPEXPIRETIME); + + public override string[] GetSingleSlotRequest() + { + var ssk = GetSingleSlotKeys; + return [ssk[0], "MEMBERS", "1", "member1"]; + } + + public override string[] GetCrossSlotRequest() => throw new NotImplementedException(); + + public override ArraySegment[] SetupSingleSlotRequest() => throw new NotImplementedException(); + } + + internal class ZPERSIST : BaseCommand + { + public override bool IsArrayCommand => false; + public override bool ArrayResponse => true; + public override string Command => nameof(ZPERSIST); + + public override string[] GetSingleSlotRequest() + { + var ssk = GetSingleSlotKeys; + return [ssk[0], "MEMBERS", "1", "member1"]; + } + + public override string[] GetCrossSlotRequest() => throw new NotImplementedException(); + + public override ArraySegment[] SetupSingleSlotRequest() => throw new NotImplementedException(); + } + + internal class ZCOLLECT : BaseCommand + { + public override bool IsArrayCommand => false; + public override bool ArrayResponse => false; + public override string Command => nameof(ZCOLLECT); + + public override string[] GetSingleSlotRequest() + { + var ssk = GetSingleSlotKeys; + return [ssk[0]]; + } + + public override string[] GetCrossSlotRequest() => throw new NotImplementedException(); + + public override ArraySegment[] SetupSingleSlotRequest() + { + var ssk = GetSingleSlotKeys; + var setup = new ArraySegment[1]; + setup[0] = new ArraySegment(["ZADD", ssk[0], "1", "a", "2", "b", "3", "c"]); + return setup; + } + } + #endregion #region HashCommands diff --git a/test/Garnet.test.cluster/RedirectTests/ClusterSlotVerificationTests.cs b/test/Garnet.test.cluster/RedirectTests/ClusterSlotVerificationTests.cs index 938d54cf44..529ae523bd 100644 --- a/test/Garnet.test.cluster/RedirectTests/ClusterSlotVerificationTests.cs +++ b/test/Garnet.test.cluster/RedirectTests/ClusterSlotVerificationTests.cs @@ -118,6 +118,16 @@ public class ClusterSlotVerificationTests new ZINTERSTORE(), new ZUNION(), new ZUNIONSTORE(), + new HEXPIRE(), + new ZPEXPIRE(), + new ZEXPIREAT(), + new ZPEXPIREAT(), + new ZTTL(), + new ZPTTL(), + new ZEXPIRETIME(), + new ZPEXPIRETIME(), + new ZPERSIST(), + new ZCOLLECT(), new HSET(), new HGET(), new HGETALL(), @@ -320,6 +330,16 @@ public virtual void OneTimeTearDown() [TestCase("ZINTERSTORE")] [TestCase("ZUNION")] [TestCase("ZUNIONSTORE")] + [TestCase("ZEXPIRE")] + [TestCase("ZPEXPIRE")] + [TestCase("ZEXPIREAT")] + [TestCase("ZPEXPIREAT")] + [TestCase("ZTTL")] + [TestCase("ZPTTL")] + [TestCase("ZEXPIRETIME")] + [TestCase("ZPEXPIRETIME")] + [TestCase("ZPERSIST")] + [TestCase("ZCOLLECT")] [TestCase("HSET")] [TestCase("HGET")] [TestCase("HGETALL")] @@ -486,6 +506,16 @@ void GarnetClientSessionClusterDown(BaseCommand command) [TestCase("ZINTERSTORE")] [TestCase("ZUNION")] [TestCase("ZUNIONSTORE")] + [TestCase("ZEXPIRE")] + [TestCase("ZPEXPIRE")] + [TestCase("ZEXPIREAT")] + [TestCase("ZPEXPIREAT")] + [TestCase("ZTTL")] + [TestCase("ZPTTL")] + [TestCase("ZEXPIRETIME")] + [TestCase("ZPEXPIRETIME")] + [TestCase("ZPERSIST")] + [TestCase("ZCOLLECT")] [TestCase("HSET")] [TestCase("HGET")] [TestCase("HGETALL")] @@ -661,6 +691,16 @@ void GarnetClientSessionOK(BaseCommand command) [TestCase("ZINTERSTORE")] [TestCase("ZUNION")] [TestCase("ZUNIONSTORE")] + [TestCase("ZEXPIRE")] + [TestCase("ZPEXPIRE")] + [TestCase("ZEXPIREAT")] + [TestCase("ZPEXPIREAT")] + [TestCase("ZTTL")] + [TestCase("ZPTTL")] + [TestCase("ZEXPIRETIME")] + [TestCase("ZPEXPIRETIME")] + [TestCase("ZPERSIST")] + [TestCase("ZCOLLECT")] [TestCase("HSET")] [TestCase("HGET")] [TestCase("HGETALL")] @@ -825,6 +865,16 @@ void GarnetClientSessionCrossslotTest(BaseCommand command) [TestCase("ZINTERSTORE")] [TestCase("ZUNION")] [TestCase("ZUNIONSTORE")] + [TestCase("ZEXPIRE")] + [TestCase("ZPEXPIRE")] + [TestCase("ZEXPIREAT")] + [TestCase("ZPEXPIREAT")] + [TestCase("ZTTL")] + [TestCase("ZPTTL")] + [TestCase("ZEXPIRETIME")] + [TestCase("ZPEXPIRETIME")] + [TestCase("ZPERSIST")] + [TestCase("ZCOLLECT")] [TestCase("HSET")] [TestCase("HGET")] [TestCase("HGETALL")] @@ -999,6 +1049,16 @@ void GarnetClientSessionMOVEDTest(BaseCommand command) [TestCase("ZINTERSTORE")] [TestCase("ZUNION")] [TestCase("ZUNIONSTORE")] + [TestCase("ZEXPIRE")] + [TestCase("ZPEXPIRE")] + [TestCase("ZEXPIREAT")] + [TestCase("ZPEXPIREAT")] + [TestCase("ZTTL")] + [TestCase("ZPTTL")] + [TestCase("ZEXPIRETIME")] + [TestCase("ZPEXPIRETIME")] + [TestCase("ZPERSIST")] + [TestCase("ZCOLLECT")] [TestCase("HSET")] [TestCase("HGET")] [TestCase("HGETALL")] @@ -1191,6 +1251,16 @@ void GarnetClientSessionASKTest(BaseCommand command) [TestCase("ZINTERSTORE")] [TestCase("ZUNION")] [TestCase("ZUNIONSTORE")] + [TestCase("ZEXPIRE")] + [TestCase("ZPEXPIRE")] + [TestCase("ZEXPIREAT")] + [TestCase("ZPEXPIREAT")] + [TestCase("ZTTL")] + [TestCase("ZPTTL")] + [TestCase("ZEXPIRETIME")] + [TestCase("ZPEXPIRETIME")] + [TestCase("ZPERSIST")] + [TestCase("ZCOLLECT")] [TestCase("HSET")] [TestCase("HGET")] [TestCase("HGETALL")] diff --git a/test/Garnet.test/Resp/ACL/RespCommandTests.cs b/test/Garnet.test/Resp/ACL/RespCommandTests.cs index f5f693e29b..09f91313b5 100644 --- a/test/Garnet.test/Resp/ACL/RespCommandTests.cs +++ b/test/Garnet.test/Resp/ACL/RespCommandTests.cs @@ -6748,6 +6748,165 @@ static async Task DoZDiffMultiAsync(GarnetClient client) } } + [Test] + public async Task ZExpireACLsAsync() + { + await CheckCommandsAsync( + "ZEXPIRE", + [DoHExpireAsync] + ); + + static async Task DoHExpireAsync(GarnetClient client) + { + var val = await client.ExecuteForStringArrayResultAsync("ZEXPIRE", ["foo", "1", "MEMBERS", "1", "bar"]); + ClassicAssert.AreEqual(1, val.Length); + ClassicAssert.AreEqual("-2", val[0]); + } + } + + [Test] + public async Task ZPExpireACLsAsync() + { + await CheckCommandsAsync( + "ZPEXPIRE", + [DoHPExpireAsync] + ); + + static async Task DoHPExpireAsync(GarnetClient client) + { + var val = await client.ExecuteForStringArrayResultAsync("ZPEXPIRE", ["foo", "1", "MEMBERS", "1", "bar"]); + ClassicAssert.AreEqual(1, val.Length); + ClassicAssert.AreEqual("-2", val[0]); + } + } + + [Test] + public async Task ZExpireAtACLsAsync() + { + await CheckCommandsAsync( + "ZEXPIREAT", + [DoHExpireAtAsync] + ); + + static async Task DoHExpireAtAsync(GarnetClient client) + { + var val = await client.ExecuteForStringArrayResultAsync("ZEXPIREAT", ["foo", DateTimeOffset.UtcNow.AddSeconds(3).ToUnixTimeSeconds().ToString(), "MEMBERS", "1", "bar"]); + ClassicAssert.AreEqual(1, val.Length); + ClassicAssert.AreEqual("-2", val[0]); + } + } + + [Test] + public async Task ZPExpireAtACLsAsync() + { + await CheckCommandsAsync( + "ZPEXPIREAT", + [DoHPExpireAtAsync] + ); + + static async Task DoHPExpireAtAsync(GarnetClient client) + { + var val = await client.ExecuteForStringArrayResultAsync("ZPEXPIREAT", ["foo", DateTimeOffset.UtcNow.AddSeconds(3).ToUnixTimeMilliseconds().ToString(), "MEMBERS", "1", "bar"]); + ClassicAssert.AreEqual(1, val.Length); + ClassicAssert.AreEqual("-2", val[0]); + } + } + + [Test] + public async Task ZExpireTimeACLsAsync() + { + await CheckCommandsAsync( + "ZEXPIRETIME", + [DoHExpireTimeAsync] + ); + + static async Task DoHExpireTimeAsync(GarnetClient client) + { + var val = await client.ExecuteForStringArrayResultAsync("ZEXPIRETIME", ["foo", "MEMBERS", "1", "bar"]); + ClassicAssert.AreEqual(1, val.Length); + ClassicAssert.AreEqual("-2", val[0]); + } + } + + [Test] + public async Task ZPExpireTimeACLsAsync() + { + await CheckCommandsAsync( + "ZPEXPIRETIME", + [DoHPExpireTimeAsync] + ); + + static async Task DoHPExpireTimeAsync(GarnetClient client) + { + var val = await client.ExecuteForStringArrayResultAsync("ZPEXPIRETIME", ["foo", "MEMBERS", "1", "bar"]); + ClassicAssert.AreEqual(1, val.Length); + ClassicAssert.AreEqual("-2", val[0]); + } + } + + [Test] + public async Task ZTTLACLsAsync() + { + await CheckCommandsAsync( + "ZTTL", + [DoHETTLAsync] + ); + + static async Task DoHETTLAsync(GarnetClient client) + { + var val = await client.ExecuteForStringArrayResultAsync("ZTTL", ["foo", "MEMBERS", "1", "bar"]); + ClassicAssert.AreEqual(1, val.Length); + ClassicAssert.AreEqual("-2", val[0]); + } + } + + [Test] + public async Task ZPTTLACLsAsync() + { + await CheckCommandsAsync( + "ZPTTL", + [DoHPETTLAsync] + ); + + static async Task DoHPETTLAsync(GarnetClient client) + { + var val = await client.ExecuteForStringArrayResultAsync("ZPTTL", ["foo", "MEMBERS", "1", "bar"]); + ClassicAssert.AreEqual(1, val.Length); + ClassicAssert.AreEqual("-2", val[0]); + } + } + + [Test] + public async Task ZPersistACLsAsync() + { + await CheckCommandsAsync( + "ZPERSIST", + [DoHPersistAsync] + ); + + static async Task DoHPersistAsync(GarnetClient client) + { + var val = await client.ExecuteForStringArrayResultAsync("ZPERSIST", ["foo", "MEMBERS", "1", "bar"]); + ClassicAssert.AreEqual(1, val.Length); + ClassicAssert.AreEqual("-2", val[0]); + } + } + + [Test] + public async Task ZCollectACLsAsync() + { + await CheckCommandsAsync( + "ZCOLLECT", + [DoHCollectAsync] + ); + + static async Task DoHCollectAsync(GarnetClient client) + { + var val = await client.ExecuteForStringResultAsync("ZCOLLECT", ["foo"]); + ClassicAssert.AreEqual("OK", val); + } + } + [Test] public async Task TimeACLsAsync() { diff --git a/website/docs/commands/data-structures.md b/website/docs/commands/data-structures.md index 3992413920..5a59451023 100644 --- a/website/docs/commands/data-structures.md +++ b/website/docs/commands/data-structures.md @@ -1523,6 +1523,241 @@ Integer reply: the number of members in the resulting sorted set at destination. --- +### ZEXPIRE + +#### Syntax + +```bash + ZEXPIRE key seconds [NX | XX | GT | LT] MEMBERS nummembers member [member ...] +``` + +Sets a timeout on one or more members of a sorted set key. After the timeout has expired, the members will automatically be deleted. The timeout is specified in seconds. + +The command supports several options to control when the expiration should be set: + +* **NX:** Only set expiry on members that have no existing expiry +* **XX:** Only set expiry on members that already have an expiry set +* **GT:** Only set expiry when it's greater than the current expiry +* **LT:** Only set expiry when it's less than the current expiry + +The **NX**, **XX**, **GT**, and **LT** options are mutually exclusive. + +#### Resp Reply + +Array reply: For each member, returns: + +* 1 if the timeout was set +* 0 if the member doesn't exist +* -1 if timeout was not set due to condition not being met + +--- + +### ZEXPIREAT + +#### Syntax + +```bash + ZEXPIREAT key unix-time-seconds [NX | XX | GT | LT] MEMBERS nummembers member [member ...] +``` + +Sets an absolute expiration time (Unix timestamp in seconds) for one or more sorted set members. After the timestamp has passed, the members will automatically be deleted. + +The command supports several options to control when the expiration should be set: + +* **NX:** Only set expiry on members that have no existing expiry +* **XX:** Only set expiry on members that already have an expiry set +* **GT:** Only set expiry when it's greater than the current expiry +* **LT:** Only set expiry when it's less than the current expiry + +The **NX**, **XX**, **GT**, and **LT** options are mutually exclusive. + +#### Resp Reply + +Array reply: For each member, returns: + +* 1 if the timeout was set +* 0 if the member doesn't exist +* -1 if timeout was not set due to condition not being met + +--- + +### ZPEXPIRE + +#### Syntax + +```bash + ZPEXPIRE key milliseconds [NX | XX | GT | LT] MEMBERS nummembers member [member ...] +``` + +Similar to HEXPIRE but the timeout is specified in milliseconds instead of seconds. + +The command supports several options to control when the expiration should be set: + +* **NX:** Only set expiry on members that have no existing expiry +* **XX:** Only set expiry on members that already have an expiry set +* **GT:** Only set expiry when it's greater than the current expiry +* **LT:** Only set expiry when it's less than the current expiry + +The **NX**, **XX**, **GT**, and **LT** options are mutually exclusive. + +#### Resp Reply + +Array reply: For each member, returns: + +* 1 if the timeout was set +* 0 if the member doesn't exist +* -1 if timeout was not set due to condition not being met + +--- + +### ZPEXPIREAT + +#### Syntax + +```bash + ZPEXPIREAT key unix-time-milliseconds [NX | XX | GT | LT] MEMBERS nummembers member [member ...] +``` + +Similar to HEXPIREAT but uses Unix timestamp in milliseconds instead of seconds. + +The command supports several options to control when the expiration should be set: + +* **NX:** Only set expiry on members that have no existing expiry +* **XX:** Only set expiry on members that already have an expiry set +* **GT:** Only set expiry when it's greater than the current expiry +* **LT:** Only set expiry when it's less than the current expiry + +The **NX**, **XX**, **GT**, and **LT** options are mutually exclusive. + +#### Resp Reply + +Array reply: For each member, returns: + +* 1 if the timeout was set +* 0 if the member doesn't exist +* -1 if timeout was not set due to condition not being met + +--- + +### ZTTL + +#### Syntax + +```bash + ZTTL key MEMBERS nummembers member [member ...] +``` + +Returns the remaining time to live in seconds for one or more sorted set members that have a timeout set. + +#### Resp Reply + +Array reply: For each member, returns: + +* TTL in seconds if the member exists and has an expiry set +* -1 if the member exists but has no expiry set +* -2 if the member does not exist + +--- + +### ZPTTL + +#### Syntax + +```bash + ZPTTL key MEMBERS nummembers member [member ...] +``` + +Similar to HTTL but returns the remaining time to live in milliseconds instead of seconds. + +#### Resp Reply + +Array reply: For each member, returns: + +* TTL in milliseconds if the member exists and has an expiry set +* -1 if the member exists but has no expiry set +* -2 if the member does not exist + +--- + +### ZEXPIRETIME + +#### Syntax + +```bash + ZEXPIRETIME key MEMBERS nummembers member [member ...] +``` + +Returns the absolute Unix timestamp (in seconds) at which the specified sorted set members will expire. + +#### Resp Reply + +Array reply: For each member, returns: + +* Unix timestamp in seconds when the member will expire +* -1 if the member exists but has no expiry set +* -2 if the member does not exist + +--- + +### ZPEXPIRETIME + +#### Syntax + +```bash + ZPEXPIRETIME key MEMBERS nummembers member [member ...] +``` + +Similar to HEXPIRETIME but returns the expiry timestamp in milliseconds instead of seconds. + +#### Resp Reply + +Array reply: For each member, returns: + +* Unix timestamp in milliseconds when the member will expire +* -1 if the member exists but has no expiry set +* -2 if the member does not exist + +--- + +### ZPERSIST + +#### Syntax + +```bash + ZPERSIST key MEMBERS nummembers member [member ...] +``` + +Removes the expiration from the specified sorted set members, making them persistent. + +#### Resp Reply + +Array reply: For each member, returns: + +* 1 if the timeout was removed +* 0 if the member exists but has no timeout +* -1 if the member does not exist + +--- + +### ZCOLLECT + +#### Syntax + +```bash + ZCOLLECT key [key ...] +``` + +Manualy trigger cleanup of expired member from memory for a given Hash set key. + +Use `*` as the key to collect it from all sorted set keys. + +#### Resp Reply + +Simple reply: OK response +Error reply: ERR ZCOLLECT scan already in progress + +--- + ## Geospatial indices ### GEOADD diff --git a/website/docs/commands/garnet-specific.md b/website/docs/commands/garnet-specific.md index 8b3a71e646..65a5bd4828 100644 --- a/website/docs/commands/garnet-specific.md +++ b/website/docs/commands/garnet-specific.md @@ -61,6 +61,25 @@ Error reply: ERR HCOLLECT scan already in progress --- +### ZCOLLECT + +#### Syntax + +```bash + ZCOLLECT key [key ...] +``` + +Manualy trigger cleanup of expired member from memory for a given Hash set key. + +Use `*` as the key to collect it from all sorted set keys. + +#### Resp Reply + +Simple reply: OK response +Error reply: ERR ZCOLLECT scan already in progress + +--- + ### COSCAN #### Syntax diff --git a/website/docs/getting-started/configuration.md b/website/docs/getting-started/configuration.md index ac0d130154..18bed2ffdb 100644 --- a/website/docs/getting-started/configuration.md +++ b/website/docs/getting-started/configuration.md @@ -120,6 +120,7 @@ For all available command line settings, run `GarnetServer.exe -h` or `GarnetSer | **AofSizeLimit** | ```--aof-size-limit``` | ```string``` | Memory size | Maximum size of AOF (rounds down to power of 2) after which unsafe truncation will be applied. Left empty AOF will grow without bound unless a checkpoint is taken | | **CompactionFrequencySecs** | ```--compaction-freq``` | ```int``` | Integer in range:
[0, MaxValue] | Background hybrid log compaction frequency in seconds. 0 = disabled (compaction performed before checkpointing instead) | | **HashCollectFrequencySecs** | ```--hcollect-freq``` | ```int``` | Integer in range:
[0, MaxValue] | Frequency in seconds for the background task to perform Hash collection. 0 = disabled. Hash collect is used to delete expired fields from hash without waiting for a write operation. Use the HCOLLECT API to collect on-demand. | +| **SortedSetCollectFrequencySecs** | ```--zcollect-freq``` | ```int``` | Integer in range:
[0, MaxValue] | Frequency in seconds for the background task to perform Sorted Set collection. 0 = disabled. Sorted Set collect is used to delete expired members from Sorted Set without waiting for a write operation. Use the ZCOLLECT API to collect on-demand. | | **CompactionType** | ```--compaction-type``` | ```LogCompactionType``` | None, Shift, Scan, Lookup | Hybrid log compaction type. Value options: None - No compaction, Shift - shift begin address without compaction (data loss), Scan - scan old pages and move live records to tail (no data loss), Lookup - lookup each record in compaction range, for record liveness checking using hash chain (no data loss) | | **CompactionForceDelete** | ```--compaction-force-delete``` | ```bool``` | | Forcefully delete the inactive segments immediately after the compaction strategy (type) is applied. If false, take a checkpoint to actually delete the older data files from disk. | | **CompactionMaxSegments** | ```--compaction-max-segments``` | ```int``` | Integer in range:
[0, MaxValue] | Number of log segments created on disk before compaction triggers. | From f0b86719f0a2dbf87ad9cd1a45f62d3d788d86aa Mon Sep 17 00:00:00 2001 From: Vijay-Nirmal Date: Wed, 5 Feb 2025 13:46:35 +0530 Subject: [PATCH 2/9] Removed temp file --- hexpire.diff | 849 --------------------------------------------------- 1 file changed, 849 deletions(-) delete mode 100644 hexpire.diff diff --git a/hexpire.diff b/hexpire.diff deleted file mode 100644 index e4d96fcbea..0000000000 --- a/hexpire.diff +++ /dev/null @@ -1,849 +0,0 @@ -diff --git a/libs/server/Objects/Hash/HashObject.cs b/libs/server/Objects/Hash/HashObject.cs -index bfa3a8b4..8ba8c21b 100644 ---- a/libs/server/Objects/Hash/HashObject.cs -+++ b/libs/server/Objects/Hash/HashObject.cs -@@ -5,6 +5,8 @@ using System; - using System.Collections.Generic; - using System.Diagnostics; - using System.IO; -+using System.Linq; -+using System.Runtime.CompilerServices; - using Garnet.common; - using Tsavorite.core; - -@@ -17,6 +19,10 @@ namespace Garnet.server - /// - public enum HashOperation : byte - { -+ HCOLLECT, -+ HEXPIRE, -+ HTTL, -+ HPERSIST, - HGET, - HMGET, - HSET, -@@ -42,6 +48,11 @@ namespace Garnet.server - public unsafe partial class HashObject : GarnetObjectBase - { - readonly Dictionary hash; -+ Dictionary expirationTimes; -+ PriorityQueue expirationQueue; -+ -+ // Byte #31 is used to denote if key has expiration (1) or not (0) -+ private const int ExpirationBitMask = 1 << 31; - - /// - /// Constructor -@@ -63,9 +74,29 @@ namespace Garnet.server - int count = reader.ReadInt32(); - for (int i = 0; i < count; i++) - { -- var item = reader.ReadBytes(reader.ReadInt32()); -+ var keyLength = reader.ReadInt32(); -+ var hasExpiration = (keyLength & ExpirationBitMask) != 0; -+ keyLength &= ~ExpirationBitMask; -+ var item = reader.ReadBytes(keyLength); - var value = reader.ReadBytes(reader.ReadInt32()); -- hash.Add(item, value); -+ -+ if (hasExpiration) -+ { -+ var expiration = reader.ReadInt64(); -+ var isExpired = expiration < DateTimeOffset.UtcNow.Ticks; -+ if (!isExpired) -+ { -+ hash.Add(item, value); -+ InitializeExpirationStructures(); -+ expirationTimes.Add(item, expiration); -+ expirationQueue.Enqueue(item, expiration); -+ UpdateExpirationSize(item, true); -+ } -+ } -+ else -+ { -+ hash.Add(item, value); -+ } - - this.UpdateSize(item, value); - } -@@ -74,10 +105,12 @@ namespace Garnet.server - /// - /// Copy constructor - /// -- public HashObject(Dictionary hash, long expiration, long size) -+ public HashObject(Dictionary hash, Dictionary expirationTimes, PriorityQueue expirationQueue, long expiration, long size) - : base(expiration, size) - { - this.hash = hash; -+ this.expirationTimes = expirationTimes; -+ this.expirationQueue = expirationQueue; - } - - /// -@@ -88,16 +121,30 @@ namespace Garnet.server - { - base.DoSerialize(writer); - -- int count = hash.Count; -+ DeleteExpiredItems(); -+ -+ int count = hash.Count; // Since expired items are already deleted, no need to worry about expiring items - writer.Write(count); - foreach (var kvp in hash) - { -+ if (expirationTimes is not null && expirationTimes.TryGetValue(kvp.Key, out var expiration)) -+ { -+ writer.Write(kvp.Key.Length | ExpirationBitMask); -+ writer.Write(kvp.Key); -+ writer.Write(kvp.Value.Length); -+ writer.Write(kvp.Value); -+ writer.Write(expiration); -+ count--; -+ continue; -+ } -+ - writer.Write(kvp.Key.Length); - writer.Write(kvp.Key); - writer.Write(kvp.Value.Length); - writer.Write(kvp.Value); - count--; - } -+ - Debug.Assert(count == 0); - } - -@@ -105,7 +152,7 @@ namespace Garnet.server - public override void Dispose() { } - - /// -- public override GarnetObjectBase Clone() => new HashObject(hash, Expiration, Size); -+ public override GarnetObjectBase Clone() => new HashObject(hash, expirationTimes, expirationQueue, Expiration, Size); - - /// - public override unsafe bool Operate(ref ObjectInput input, ref SpanByteAndMemory output, out long sizeChange, out bool removeKey) -@@ -152,6 +199,15 @@ namespace Garnet.server - case HashOperation.HEXISTS: - HashExists(ref input, _output); - break; -+ case HashOperation.HEXPIRE: -+ HashExpire(ref input, ref output); -+ break; -+ case HashOperation.HTTL: -+ HashTimeToLive(ref input, ref output); -+ break; -+ case HashOperation.HPERSIST: -+ HashPersist(ref input, ref output); -+ break; - case HashOperation.HKEYS: - HashGetKeysOrValues(ref input, ref output); - break; -@@ -170,6 +226,9 @@ namespace Garnet.server - case HashOperation.HRANDFIELD: - HashRandomField(ref input, ref output); - break; -+ case HashOperation.HCOLLECT: -+ HashCollect(ref input, _output); -+ break; - case HashOperation.HSCAN: - if (ObjectUtils.ReadScanInput(ref input, ref output, out var cursorInput, out var pattern, - out var patternLength, out var limitCount, out bool isNoValue, out var error)) -@@ -202,6 +261,38 @@ namespace Garnet.server - Debug.Assert(this.Size >= MemoryUtils.DictionaryOverhead); - } - -+ [MethodImpl(MethodImplOptions.AggressiveInlining)] -+ private void InitializeExpirationStructures() -+ { -+ if (expirationTimes is null) -+ { -+ expirationTimes = new Dictionary(ByteArrayComparer.Instance); -+ expirationQueue = new PriorityQueue(); -+ this.Size += MemoryUtils.DictionaryOverhead + MemoryUtils.PriorityQueueOverhead; -+ } -+ } -+ -+ [MethodImpl(MethodImplOptions.AggressiveInlining)] -+ private void UpdateExpirationSize(ReadOnlySpan key, bool add = true) -+ { -+ // Account for dictionary entry and priority queue entry -+ var size = IntPtr.Size + sizeof(long) + MemoryUtils.DictionaryEntryOverhead -+ + IntPtr.Size + sizeof(long) + MemoryUtils.PriorityQueueEntryOverhead; -+ this.Size += add ? size : -size; -+ } -+ -+ [MethodImpl(MethodImplOptions.AggressiveInlining)] -+ private void CleanupExpirationStructures() -+ { -+ if (expirationTimes.Count == 0) -+ { -+ this.Size -= (IntPtr.Size + sizeof(long) + MemoryUtils.PriorityQueueOverhead) * expirationQueue.Count; -+ this.Size -= MemoryUtils.DictionaryOverhead + MemoryUtils.PriorityQueueOverhead; -+ expirationTimes = null; -+ expirationQueue = null; -+ } -+ } -+ - /// - public override unsafe void Scan(long start, out List items, out long cursor, int count = 10, byte* pattern = default, int patternLength = 0, bool isNoValue = false) - { -@@ -217,8 +308,15 @@ namespace Garnet.server - // Hashset has key and value, so count is multiplied by 2 - count = isNoValue ? count : count * 2; - int index = 0; -+ var expiredKeysCount = 0; - foreach (var item in hash) - { -+ if (IsExpired(item.Key)) -+ { -+ expiredKeysCount++; -+ continue; -+ } -+ - if (index < start) - { - index++; -@@ -256,8 +354,241 @@ namespace Garnet.server - } - - // Indicates end of collection has been reached. -- if (cursor == hash.Count) -+ if (cursor + expiredKeysCount == hash.Count) - cursor = 0; - } -+ -+ [MethodImpl(MethodImplOptions.AggressiveInlining)] -+ private bool IsExpired(byte[] key) => expirationTimes is not null && expirationTimes.TryGetValue(key, out var expiration) && expiration < DateTimeOffset.UtcNow.Ticks; -+ -+ private void DeleteExpiredItems() -+ { -+ if (expirationTimes is null) -+ return; -+ -+ while (expirationQueue.TryPeek(out var key, out var expiration) && expiration < DateTimeOffset.UtcNow.Ticks) -+ { -+ // expirationTimes and expirationQueue will be out of sync when user is updating the expire time of key which already has some TTL. -+ // PriorityQueue Doesn't have update option, so we will just enqueue the new expiration and already treat expirationTimes as the source of truth -+ if (expirationTimes.TryGetValue(key, out var actualExpiration) && actualExpiration == expiration) -+ { -+ expirationTimes.Remove(key); -+ expirationQueue.Dequeue(); -+ UpdateExpirationSize(key, false); -+ if (hash.TryGetValue(key, out var value)) -+ { -+ hash.Remove(key); -+ UpdateSize(key, value, false); -+ } -+ } -+ else -+ { -+ expirationQueue.Dequeue(); -+ this.Size -= MemoryUtils.PriorityQueueEntryOverhead + IntPtr.Size + sizeof(long); -+ } -+ } -+ -+ CleanupExpirationStructures(); -+ } -+ -+ private bool TryGetValue(byte[] key, out byte[] value) -+ { -+ value = default; -+ if (IsExpired(key)) -+ { -+ return false; -+ } -+ return hash.TryGetValue(key, out value); -+ } -+ -+ private bool Remove(byte[] key, out byte[] value) -+ { -+ DeleteExpiredItems(); -+ var result = hash.Remove(key, out value); -+ if (result) -+ { -+ UpdateSize(key, value, false); -+ } -+ return result; -+ } -+ -+ private int Count() -+ { -+ if (expirationTimes is null) -+ { -+ return hash.Count; -+ } -+ -+ var expiredKeysCount = 0; -+ foreach (var item in expirationTimes) -+ { -+ if (IsExpired(item.Key)) -+ { -+ expiredKeysCount++; -+ } -+ } -+ -+ return hash.Count - expiredKeysCount; -+ } -+ -+ [MethodImpl(MethodImplOptions.AggressiveInlining)] -+ private bool HasExpirableItems() -+ { -+ return expirationTimes is not null; -+ } -+ -+ private bool ContainsKey(byte[] key) -+ { -+ var result = hash.ContainsKey(key); -+ if (result && IsExpired(key)) -+ { -+ return false; -+ } -+ -+ return result; -+ } -+ -+ [MethodImpl(MethodImplOptions.AggressiveInlining)] -+ private void Add(byte[] key, byte[] value) -+ { -+ DeleteExpiredItems(); -+ hash.Add(key, value); -+ UpdateSize(key, value); -+ } -+ -+ private void Set(byte[] key, byte[] value) -+ { -+ DeleteExpiredItems(); -+ hash[key] = value; -+ // Skip overhead as existing item is getting replaced. -+ this.Size += Utility.RoundUp(value.Length, IntPtr.Size) - -+ Utility.RoundUp(value.Length, IntPtr.Size); -+ -+ // To persist the key, if it has an expiration -+ if (expirationTimes is not null && expirationTimes.TryGetValue(key, out var currentExpiration)) -+ { -+ expirationTimes.Remove(key); -+ this.Size -= IntPtr.Size + sizeof(long) + MemoryUtils.DictionaryEntryOverhead; -+ CleanupExpirationStructures(); -+ } -+ } -+ -+ private void SetWithoutPersist(byte[] key, byte[] value) -+ { -+ DeleteExpiredItems(); -+ hash[key] = value; -+ // Skip overhead as existing item is getting replaced. -+ this.Size += Utility.RoundUp(value.Length, IntPtr.Size) - -+ Utility.RoundUp(value.Length, IntPtr.Size); -+ } -+ -+ private int SetExpiration(byte[] key, long expiration, ExpireOption expireOption) -+ { -+ if (!ContainsKey(key)) -+ { -+ return (int)ExpireResult.KeyNotFound; -+ } -+ -+ if (expiration <= DateTimeOffset.UtcNow.Ticks) -+ { -+ Remove(key, out _); -+ return (int)ExpireResult.KeyAlreadyExpired; -+ } -+ -+ InitializeExpirationStructures(); -+ -+ if (expirationTimes.TryGetValue(key, out var currentExpiration)) -+ { -+ if (expireOption.HasFlag(ExpireOption.NX) || -+ (expireOption.HasFlag(ExpireOption.GT) && expiration <= currentExpiration) || -+ (expireOption.HasFlag(ExpireOption.LT) && expiration >= currentExpiration)) -+ { -+ return (int)ExpireResult.ExpireConditionNotMet; -+ } -+ -+ expirationTimes[key] = expiration; -+ expirationQueue.Enqueue(key, expiration); -+ // Size of dictionary entry already accounted for as the key already exists -+ this.Size += IntPtr.Size + sizeof(long) + MemoryUtils.PriorityQueueEntryOverhead; -+ } -+ else -+ { -+ if (expireOption.HasFlag(ExpireOption.XX) || expireOption.HasFlag(ExpireOption.GT)) -+ { -+ return (int)ExpireResult.ExpireConditionNotMet; -+ } -+ -+ expirationTimes[key] = expiration; -+ expirationQueue.Enqueue(key, expiration); -+ UpdateExpirationSize(key); -+ } -+ -+ return (int)ExpireResult.ExpireUpdated; -+ } -+ -+ private int Persist(byte[] key) -+ { -+ if (!ContainsKey(key)) -+ { -+ return -2; -+ } -+ -+ if (expirationTimes is not null && expirationTimes.TryGetValue(key, out var currentExpiration)) -+ { -+ expirationTimes.Remove(key); -+ this.Size -= IntPtr.Size + sizeof(long) + MemoryUtils.DictionaryEntryOverhead; -+ CleanupExpirationStructures(); -+ return 1; -+ } -+ -+ return -1; -+ } -+ -+ private long GetExpiration(byte[] key) -+ { -+ if (!ContainsKey(key)) -+ { -+ return -2; -+ } -+ -+ if (expirationTimes.TryGetValue(key, out var expiration)) -+ { -+ return expiration; -+ } -+ -+ return -1; -+ } -+ -+ private KeyValuePair ElementAt(int index) -+ { -+ if (HasExpirableItems()) -+ { -+ var currIndex = 0; -+ foreach (var item in hash) -+ { -+ if (IsExpired(item.Key)) -+ { -+ continue; -+ } -+ -+ if (currIndex++ == index) -+ { -+ return item; -+ } -+ } -+ -+ throw new ArgumentOutOfRangeException("index is outside the bounds of the source sequence."); -+ } -+ -+ return hash.ElementAt(index); -+ } -+ } -+ -+ enum ExpireResult -+ { -+ KeyNotFound = -2, -+ ExpireConditionNotMet = 0, -+ ExpireUpdated = 1, -+ KeyAlreadyExpired = 2, - } - } -\ No newline at end of file -diff --git a/libs/server/Objects/Hash/HashObjectImpl.cs b/libs/server/Objects/Hash/HashObjectImpl.cs -index 674aebfd..14f6a84a 100644 ---- a/libs/server/Objects/Hash/HashObjectImpl.cs -+++ b/libs/server/Objects/Hash/HashObjectImpl.cs -@@ -33,7 +33,7 @@ namespace Garnet.server - { - var key = input.parseState.GetArgSliceByRef(0).SpanByte.ToByteArray(); - -- if (hash.TryGetValue(key, out var hashValue)) -+ if (TryGetValue(key, out var hashValue)) - { - while (!RespWriteUtils.WriteBulkString(hashValue, ref curr, end)) - ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); -@@ -75,7 +75,7 @@ namespace Garnet.server - { - var key = input.parseState.GetArgSliceByRef(i).SpanByte.ToByteArray(); - -- if (hash.TryGetValue(key, out var hashValue)) -+ if (TryGetValue(key, out var hashValue)) - { - while (!RespWriteUtils.WriteBulkString(hashValue, ref curr, end)) - ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); -@@ -115,17 +115,24 @@ namespace Garnet.server - { - if (respProtocolVersion < 3) - { -- while (!RespWriteUtils.WriteArrayLength(hash.Count * 2, ref curr, end)) -+ while (!RespWriteUtils.WriteArrayLength(Count() * 2, ref curr, end)) - ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); - } - else - { -- while (!RespWriteUtils.WriteMapLength(hash.Count, ref curr, end)) -+ while (!RespWriteUtils.WriteMapLength(Count(), ref curr, end)) - ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); - } - -+ var isExpirable = HasExpirableItems(); -+ - foreach (var item in hash) - { -+ if (isExpirable && IsExpired(item.Key)) -+ { -+ continue; -+ } -+ - while (!RespWriteUtils.WriteBulkString(item.Key, ref curr, end)) - ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); - while (!RespWriteUtils.WriteBulkString(item.Value, ref curr, end)) -@@ -151,17 +158,16 @@ namespace Garnet.server - { - var key = input.parseState.GetArgSliceByRef(i).SpanByte.ToByteArray(); - -- if (hash.Remove(key, out var hashValue)) -+ if (Remove(key, out var hashValue)) - { - _output->result1++; -- this.UpdateSize(key, hashValue, false); - } - } - } - - private void HashLength(byte* output) - { -- ((ObjectOutputHeader*)output)->result1 = hash.Count; -+ ((ObjectOutputHeader*)output)->result1 = Count(); - } - - private void HashStrLength(ref ObjectInput input, byte* output) -@@ -170,7 +176,7 @@ namespace Garnet.server - *_output = default; - - var key = input.parseState.GetArgSliceByRef(0).SpanByte.ToByteArray(); -- _output->result1 = hash.TryGetValue(key, out var hashValue) ? hashValue.Length : 0; -+ _output->result1 = TryGetValue(key, out var hashValue) ? hashValue.Length : 0; - } - - private void HashExists(ref ObjectInput input, byte* output) -@@ -179,7 +185,7 @@ namespace Garnet.server - *_output = default; - - var field = input.parseState.GetArgSliceByRef(0).SpanByte.ToByteArray(); -- _output->result1 = hash.ContainsKey(field) ? 1 : 0; -+ _output->result1 = ContainsKey(field) ? 1 : 0; - } - - private void HashRandomField(ref ObjectInput input, ref SpanByteAndMemory output) -@@ -204,11 +210,21 @@ namespace Garnet.server - { - if (includedCount) - { -- if (countParameter > 0 && countParameter > hash.Count) -- countParameter = hash.Count; -+ var count = Count(); -+ -+ if (count == 0) // This can happen because of expiration but RMW operation haven't applied yet -+ { -+ while (!RespWriteUtils.WriteEmptyArray(ref curr, end)) -+ ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); -+ _output.result1 = 0; -+ return; -+ } -+ -+ if (countParameter > 0 && countParameter > count) -+ countParameter = count; - - var absCount = Math.Abs(countParameter); -- var indexes = RandomUtils.PickKRandomIndexes(hash.Count, absCount, seed, countParameter > 0); -+ var indexes = RandomUtils.PickKRandomIndexes(count, absCount, seed, countParameter > 0); - - // Write the size of the array reply - while (!RespWriteUtils.WriteArrayLength(withValues ? absCount * 2 : absCount, ref curr, end)) -@@ -216,7 +232,7 @@ namespace Garnet.server - - foreach (var index in indexes) - { -- var pair = hash.ElementAt(index); -+ var pair = ElementAt(index); - while (!RespWriteUtils.WriteBulkString(pair.Key, ref curr, end)) - ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); - -@@ -232,8 +248,17 @@ namespace Garnet.server - else // No count parameter is present, we just return a random field - { - // Write a bulk string value of a random field from the hash value stored at key. -- var index = RandomUtils.PickRandomIndex(hash.Count, seed); -- var pair = hash.ElementAt(index); -+ var count = Count(); -+ if (count == 0) // This can happen because of expiration but RMW operation haven't applied yet -+ { -+ while (!RespWriteUtils.WriteNull(ref curr, end)) -+ ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); -+ _output.result1 = 0; -+ return; -+ } -+ -+ var index = RandomUtils.PickRandomIndex(count, seed); -+ var pair = ElementAt(index); - while (!RespWriteUtils.WriteBulkString(pair.Key, ref curr, end)) - ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); - countDone = 1; -@@ -262,26 +287,31 @@ namespace Garnet.server - var key = input.parseState.GetArgSliceByRef(i).SpanByte.ToByteArray(); - var value = input.parseState.GetArgSliceByRef(i + 1).SpanByte.ToByteArray(); - -- if (!hash.TryGetValue(key, out var hashValue)) -+ if (!TryGetValue(key, out var hashValue)) - { -- hash.Add(key, value); -- this.UpdateSize(key, value); -+ Add(key, value); - _output->result1++; - } -- else if ((hop == HashOperation.HSET || hop == HashOperation.HMSET) && hashValue != default && -- !hashValue.AsSpan().SequenceEqual(value)) -+ else if ((hop == HashOperation.HSET || hop == HashOperation.HMSET) && hashValue != default) - { -- hash[key] = value; -- // Skip overhead as existing item is getting replaced. -- this.Size += Utility.RoundUp(value.Length, IntPtr.Size) - -- Utility.RoundUp(hashValue.Length, IntPtr.Size); -+ Set(key, value); - } - } - } - -+ private void HashCollect(ref ObjectInput input, byte* output) -+ { -+ var _output = (ObjectOutputHeader*)output; -+ *_output = default; -+ -+ DeleteExpiredItems(); -+ -+ _output->result1 = 1; -+ } -+ - private void HashGetKeysOrValues(ref ObjectInput input, ref SpanByteAndMemory output) - { -- var count = hash.Count; -+ var count = Count(); - var op = input.header.HashOp; - - var isMemory = false; -@@ -297,8 +327,15 @@ namespace Garnet.server - while (!RespWriteUtils.WriteArrayLength(count, ref curr, end)) - ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); - -+ var isExpirable = HasExpirableItems(); -+ - foreach (var item in hash) - { -+ if (isExpirable && IsExpired(item.Key)) -+ { -+ continue; -+ } -+ - if (HashOperation.HKEYS == op) - { - while (!RespWriteUtils.WriteBulkString(item.Key, ref curr, end)) -@@ -343,7 +380,7 @@ namespace Garnet.server - var key = input.parseState.GetArgSliceByRef(0).SpanByte.ToByteArray(); - var incrSlice = input.parseState.GetArgSliceByRef(1); - -- var valueExists = hash.TryGetValue(key, out var value); -+ var valueExists = TryGetValue(key, out var value); - if (op == HashOperation.HINCRBY) - { - if (!NumUtils.TryParse(incrSlice.ReadOnlySpan, out int incr)) -@@ -376,15 +413,12 @@ namespace Garnet.server - resultSpan = resultSpan.Slice(0, bytesWritten); - - resultBytes = resultSpan.ToArray(); -- hash[key] = resultBytes; -- Size += Utility.RoundUp(resultBytes.Length, IntPtr.Size) - -- Utility.RoundUp(value.Length, IntPtr.Size); -+ SetWithoutPersist(key, resultBytes); - } - else - { - resultBytes = incrSlice.SpanByte.ToByteArray(); -- hash.Add(key, resultBytes); -- UpdateSize(key, resultBytes); -+ Add(key, resultBytes); - } - - while (!RespWriteUtils.WriteIntegerFromBytes(resultBytes, ref curr, end)) -@@ -417,15 +451,12 @@ namespace Garnet.server - result += incr; - - resultBytes = Encoding.ASCII.GetBytes(result.ToString(CultureInfo.InvariantCulture)); -- hash[key] = resultBytes; -- Size += Utility.RoundUp(resultBytes.Length, IntPtr.Size) - -- Utility.RoundUp(value.Length, IntPtr.Size); -+ SetWithoutPersist(key, resultBytes); - } - else - { - resultBytes = incrSlice.SpanByte.ToByteArray(); -- hash.Add(key, resultBytes); -- UpdateSize(key, resultBytes); -+ Add(key, resultBytes); - } - - while (!RespWriteUtils.WriteBulkString(resultBytes, ref curr, end)) -@@ -444,5 +475,138 @@ namespace Garnet.server - output.Length = (int)(curr - ptr); - } - } -+ -+ private void HashExpire(ref ObjectInput input, ref SpanByteAndMemory output) -+ { -+ var isMemory = false; -+ MemoryHandle ptrHandle = default; -+ var ptr = output.SpanByte.ToPointer(); -+ -+ var curr = ptr; -+ var end = curr + output.Length; -+ -+ ObjectOutputHeader _output = default; -+ try -+ { -+ DeleteExpiredItems(); -+ -+ var expireOption = (ExpireOption)input.arg1; -+ var expiration = input.parseState.GetLong(0); -+ var numFields = input.parseState.Count - 1; -+ while (!RespWriteUtils.WriteArrayLength(numFields, ref curr, end)) -+ ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); -+ -+ foreach (var item in input.parseState.Parameters.Slice(1)) -+ { -+ var result = SetExpiration(item.ToArray(), expiration, expireOption); -+ while (!RespWriteUtils.WriteInteger(result, ref curr, end)) -+ ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); -+ _output.result1++; -+ } -+ } -+ finally -+ { -+ while (!RespWriteUtils.WriteDirect(ref _output, ref curr, end)) -+ ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); -+ -+ if (isMemory) ptrHandle.Dispose(); -+ output.Length = (int)(curr - ptr); -+ } -+ } -+ -+ private void HashTimeToLive(ref ObjectInput input, ref SpanByteAndMemory output) -+ { -+ var isMemory = false; -+ MemoryHandle ptrHandle = default; -+ var ptr = output.SpanByte.ToPointer(); -+ -+ var curr = ptr; -+ var end = curr + output.Length; -+ -+ ObjectOutputHeader _output = default; -+ try -+ { -+ DeleteExpiredItems(); -+ -+ var isMilliseconds = input.arg1 == 1; -+ var isTimestamp = input.arg2 == 1; -+ var numFields = input.parseState.Count; -+ while (!RespWriteUtils.WriteArrayLength(numFields, ref curr, end)) -+ ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); -+ -+ foreach (var item in input.parseState.Parameters) -+ { -+ var result = GetExpiration(item.ToArray()); -+ -+ if (result >= 0) -+ { -+ if (isTimestamp && isMilliseconds) -+ { -+ result = ConvertUtils.UnixTimeInMillisecondsFromTicks(result); -+ } -+ else if (isTimestamp && !isMilliseconds) -+ { -+ result = ConvertUtils.UnixTimeInSecondsFromTicks(result); -+ } -+ else if (!isTimestamp && isMilliseconds) -+ { -+ result = ConvertUtils.MillisecondsFromDiffUtcNowTicks(result); -+ } -+ else if (!isTimestamp && !isMilliseconds) -+ { -+ result = ConvertUtils.SecondsFromDiffUtcNowTicks(result); -+ } -+ } -+ -+ while (!RespWriteUtils.WriteInteger(result, ref curr, end)) -+ ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); -+ _output.result1++; -+ } -+ } -+ finally -+ { -+ while (!RespWriteUtils.WriteDirect(ref _output, ref curr, end)) -+ ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); -+ -+ if (isMemory) ptrHandle.Dispose(); -+ output.Length = (int)(curr - ptr); -+ } -+ } -+ -+ private void HashPersist(ref ObjectInput input, ref SpanByteAndMemory output) -+ { -+ var isMemory = false; -+ MemoryHandle ptrHandle = default; -+ var ptr = output.SpanByte.ToPointer(); -+ -+ var curr = ptr; -+ var end = curr + output.Length; -+ -+ ObjectOutputHeader _output = default; -+ try -+ { -+ DeleteExpiredItems(); -+ -+ var numFields = input.parseState.Count; -+ while (!RespWriteUtils.WriteArrayLength(numFields, ref curr, end)) -+ ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); -+ -+ foreach (var item in input.parseState.Parameters) -+ { -+ var result = Persist(item.ToArray()); -+ while (!RespWriteUtils.WriteInteger(result, ref curr, end)) -+ ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); -+ _output.result1++; -+ } -+ } -+ finally -+ { -+ while (!RespWriteUtils.WriteDirect(ref _output, ref curr, end)) -+ ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); -+ -+ if (isMemory) ptrHandle.Dispose(); -+ output.Length = (int)(curr - ptr); -+ } -+ } - } - } -\ No newline at end of file -diff --git a/libs/server/Objects/Types/GarnetObject.cs b/libs/server/Objects/Types/GarnetObject.cs -index f5366dff..7474d547 100644 ---- a/libs/server/Objects/Types/GarnetObject.cs -+++ b/libs/server/Objects/Types/GarnetObject.cs -@@ -66,6 +66,12 @@ namespace Garnet.server - SetOperation.SPOP => false, - _ => true, - }, -+ GarnetObjectType.Hash => header.HashOp switch -+ { -+ HashOperation.HEXPIRE => false, -+ HashOperation.HCOLLECT => false, -+ _ => true, -+ }, - GarnetObjectType.Expire => false, - GarnetObjectType.PExpire => false, - GarnetObjectType.Persist => false, From d0e969739e3ee284a236d9c58af37fde9f8bf5c7 Mon Sep 17 00:00:00 2001 From: Vijay-Nirmal Date: Mon, 10 Feb 2025 01:51:23 +0530 Subject: [PATCH 3/9] Added test cases --- .../ItemBroker/CollectionItemBroker.cs | 6 +- .../Objects/SortedSet/SortedSetObject.cs | 140 +- .../Objects/SortedSet/SortedSetObjectImpl.cs | 17 +- libs/server/Resp/AdminCommands.cs | 2 +- libs/server/Resp/Parser/RespCommand.cs | 24 +- .../Session/ObjectStore/SortedSetOps.cs | 19 +- .../RespBlockingCollectionTests.cs | 36 + test/Garnet.test/RespSortedSetTests.cs | 1364 ++++++++++++++++- 8 files changed, 1522 insertions(+), 86 deletions(-) diff --git a/libs/server/Objects/ItemBroker/CollectionItemBroker.cs b/libs/server/Objects/ItemBroker/CollectionItemBroker.cs index 044a698bda..289ea1b0ff 100644 --- a/libs/server/Objects/ItemBroker/CollectionItemBroker.cs +++ b/libs/server/Objects/ItemBroker/CollectionItemBroker.cs @@ -403,7 +403,7 @@ private static unsafe bool TryGetNextSetObjects(byte[] key, SortedSetObject sort { result = default; - if (sortedSetObj.Dictionary.Count == 0) return false; + if (sortedSetObj.Count() == 0) return false; switch (command) { @@ -416,7 +416,7 @@ private static unsafe bool TryGetNextSetObjects(byte[] key, SortedSetObject sort case RespCommand.BZMPOP: var lowScoresFirst = *(bool*)cmdArgs[0].ptr; var popCount = *(int*)cmdArgs[1].ptr; - popCount = Math.Min(popCount, sortedSetObj.Dictionary.Count); + popCount = Math.Min(popCount, sortedSetObj.Count()); var scores = new double[popCount]; var items = new byte[popCount][]; @@ -546,7 +546,7 @@ private unsafe bool TryGetResult(byte[] key, StorageSession storageSession, Resp return false; } case SortedSetObject setObj: - currCount = setObj.Dictionary.Count; + currCount = setObj.Count(); if (objectType != GarnetObjectType.SortedSet) return false; if (currCount == 0) diff --git a/libs/server/Objects/SortedSet/SortedSetObject.cs b/libs/server/Objects/SortedSet/SortedSetObject.cs index 1f29b17d96..48cf6cb809 100644 --- a/libs/server/Objects/SortedSet/SortedSetObject.cs +++ b/libs/server/Objects/SortedSet/SortedSetObject.cs @@ -190,9 +190,28 @@ public SortedSetObject(SortedSet<(double, byte[])> sortedSet, Dictionary (byte)GarnetObjectType.SortedSet; /// - /// Get sorted set as a dictionary + /// Get sorted set as a dictionary. /// - public Dictionary Dictionary => sortedSetDict; + public Dictionary Dictionary + { + get + { + if (!HasExpirableItems() || (expirationQueue.TryPeek(out _, out var expiration) && expiration > DateTimeOffset.UtcNow.Ticks)) + { + return sortedSetDict; + } + + var result = new Dictionary(ByteArrayComparer.Instance); + foreach (var kvp in sortedSetDict) + { + if (!IsExpired(kvp.Key)) + { + result.Add(kvp.Key, kvp.Value); + } + } + return result; + } + } /// /// Serialize @@ -412,14 +431,14 @@ public override unsafe void Scan(long start, out List items, out long cu int index = 0; - if (Dictionary.Count < start) + if (sortedSetDict.Count < start) { cursor = 0; return; } var expiredKeysCount = 0; - foreach (var item in Dictionary) + foreach (var item in sortedSetDict) { if (IsExpired(item.Key)) { @@ -478,18 +497,32 @@ public override unsafe void Scan(long start, out List items, out long cu /// /// Compute difference of two dictionaries, with new result /// - public static Dictionary CopyDiff(Dictionary dict1, Dictionary dict2) + public static Dictionary CopyDiff(SortedSetObject sortedSetObject1, SortedSetObject sortedSetObject2) { - if (dict1 == null) - return []; + if (sortedSetObject1 == null) + return new Dictionary(ByteArrayComparer.Instance); - if (dict2 == null) - return new Dictionary(dict1, dict1.Comparer); + if (sortedSetObject2 == null) + { + if (sortedSetObject1.expirationTimes is null) + { + return new Dictionary(sortedSetObject1.sortedSetDict, ByteArrayComparer.Instance); + } + else + { + var directResult = new Dictionary(ByteArrayComparer.Instance); + foreach (var item in sortedSetObject1.sortedSetDict) + { + if (!sortedSetObject1.IsExpired(item.Key)) + directResult.Add(item.Key, item.Value); + } + } + } - var result = new Dictionary(dict1.Comparer); - foreach (var item in dict1) + var result = new Dictionary(ByteArrayComparer.Instance); + foreach (var item in sortedSetObject1.sortedSetDict) { - if (!dict2.ContainsKey(item.Key)) + if (!sortedSetObject1.IsExpired(item.Key) && !sortedSetObject2.IsExpired(item.Key) && !sortedSetObject2.sortedSetDict.ContainsKey(item.Key)) result.Add(item.Key, item.Value); } return result; @@ -498,20 +531,59 @@ public static Dictionary CopyDiff(Dictionary dic /// /// Remove keys existing in second dictionary, from the first dictionary, if they exist /// - public static void InPlaceDiff(Dictionary dict1, Dictionary dict2) + public static void InPlaceDiff(Dictionary dict1, SortedSetObject sortedSetObject2) { Debug.Assert(dict1 != null); - if (dict2 != null) + if (sortedSetObject2 != null) { foreach (var item in dict1) { - if (dict2.ContainsKey(item.Key)) + if (!sortedSetObject2.IsExpired(item.Key) && sortedSetObject2.sortedSetDict.ContainsKey(item.Key)) dict1.Remove(item.Key); } } } + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public bool TryGetScore(byte[] key, out double value) + { + value = default; + if (IsExpired(key)) + { + return false; + } + + return sortedSetDict.TryGetValue(key, out value); + } + + public int Count() + { + if (expirationTimes is null) + { + return sortedSetDict.Count; + } + var expiredKeysCount = 0; + + foreach (var item in expirationTimes) + { + if (IsExpired(item.Key)) + { + expiredKeysCount++; + } + } + return sortedSetDict.Count - expiredKeysCount; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public bool IsExpired(byte[] key) => expirationTimes is not null && expirationTimes.TryGetValue(key, out var expiration) && expiration < DateTimeOffset.UtcNow.Ticks; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public bool HasExpirableItems() + { + return expirationTimes is not null; + } + #endregion private void InitializeExpirationStructures() { @@ -572,38 +644,6 @@ private void DeleteExpiredItems() CleanupExpirationStructures(); } - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private bool TryGetScore(byte[] key, out double value) - { - value = default; - if (IsExpired(key)) - { - return false; - } - - return sortedSetDict.TryGetValue(key, out value); - } - - private int Count() - { - if (expirationTimes is null) - { - return sortedSetDict.Count; - } - var expiredKeysCount = 0; - foreach (var item in expirationTimes) - { - if (IsExpired(item.Key)) - { - expiredKeysCount++; - } - } - return sortedSetDict.Count - expiredKeysCount; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private bool IsExpired(byte[] key) => expirationTimes is not null && expirationTimes.TryGetValue(key, out var expiration) && expiration < DateTimeOffset.UtcNow.Ticks; - private int SetExpiration(byte[] key, long expiration, ExpireOption expireOption) { if (!sortedSetDict.ContainsKey(key)) @@ -688,12 +728,6 @@ private long GetExpiration(byte[] key) return -1; } - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private bool HasExpirableItems() - { - return expirationTimes is not null; - } - private KeyValuePair ElementAt(int index) { if (HasExpirableItems()) diff --git a/libs/server/Objects/SortedSet/SortedSetObjectImpl.cs b/libs/server/Objects/SortedSet/SortedSetObjectImpl.cs index 0a74e80b46..60ffec5f41 100644 --- a/libs/server/Objects/SortedSet/SortedSetObjectImpl.cs +++ b/libs/server/Objects/SortedSet/SortedSetObjectImpl.cs @@ -157,7 +157,10 @@ private void SortedSetAdd(ref ObjectInput input, ref SpanByteAndMemory output) // No need for update if (score == scoreStored) + { + Persist(member); continue; + } // Don't update existing member if NX flag is set // or if GT/LT flag is set and existing score is higher/lower than new score, respectively @@ -533,7 +536,9 @@ private void SortedSetRange(ref ObjectInput input, ref SpanByteAndMemory output) WriteSortedSetResult(options.WithScores, scoredElements.Count, respProtocolVersion, scoredElements, ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); } else - { // byIndex + { + // byIndex + var setCount = Count(); int minIndex = (int)minValue, maxIndex = (int)maxValue; if (options.ValidLimit) { @@ -541,7 +546,7 @@ private void SortedSetRange(ref ObjectInput input, ref SpanByteAndMemory output) ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); return; } - else if (minValue > sortedSetDict.Count - 1) + else if (minValue > setCount - 1) { // return empty list while (!RespWriteUtils.TryWriteEmptyArray(ref curr, end)) @@ -553,15 +558,15 @@ private void SortedSetRange(ref ObjectInput input, ref SpanByteAndMemory output) //shift from the end of the set if (minIndex < 0) { - minIndex = sortedSetDict.Count + minIndex; + minIndex = setCount + minIndex; } if (maxIndex < 0) { - maxIndex = sortedSetDict.Count + maxIndex; + maxIndex = setCount + maxIndex; } - else if (maxIndex >= sortedSetDict.Count) + else if (maxIndex >= setCount) { - maxIndex = sortedSetDict.Count - 1; + maxIndex = setCount - 1; } // No elements to return if both indexes fall outside the range or min is higher than max diff --git a/libs/server/Resp/AdminCommands.cs b/libs/server/Resp/AdminCommands.cs index f9fbfeac84..202f7de6fa 100644 --- a/libs/server/Resp/AdminCommands.cs +++ b/libs/server/Resp/AdminCommands.cs @@ -639,7 +639,7 @@ private bool NetworkHCOLLECT(ref TGarnetApi storageApi) return true; } - private bool NetworZHCOLLECT(ref TGarnetApi storageApi) + private bool NetworkZCOLLECT(ref TGarnetApi storageApi) where TGarnetApi : IGarnetApi { if (parseState.Count < 1) diff --git a/libs/server/Resp/Parser/RespCommand.cs b/libs/server/Resp/Parser/RespCommand.cs index e0e10df058..2a4148793c 100644 --- a/libs/server/Resp/Parser/RespCommand.cs +++ b/libs/server/Resp/Parser/RespCommand.cs @@ -865,10 +865,6 @@ private RespCommand FastParseArrayCommand(ref int count, ref ReadOnlySpan { return RespCommand.HTTL; } - else if (*(ulong*)(ptr + 2) == MemoryMarshal.Read("\r\nZTTL\r\n"u8)) - { - return RespCommand.ZTTL; - } break; case 'K': @@ -954,6 +950,10 @@ private RespCommand FastParseArrayCommand(ref int count, ref ReadOnlySpan { return RespCommand.ZREM; } + else if (*(ulong*)(ptr + 2) == MemoryMarshal.Read("\r\nZTTL\r\n"u8)) + { + return RespCommand.ZTTL; + } break; } break; @@ -1047,10 +1047,6 @@ private RespCommand FastParseArrayCommand(ref int count, ref ReadOnlySpan { return RespCommand.HPTTL; } - else if (*(ulong*)(ptr + 3) == MemoryMarshal.Read("\nZPTTL\r\n"u8)) - { - return RespCommand.ZPTTL; - } break; case 'L': @@ -1133,6 +1129,10 @@ private RespCommand FastParseArrayCommand(ref int count, ref ReadOnlySpan { return RespCommand.ZMPOP; } + else if (*(ulong*)(ptr + 3) == MemoryMarshal.Read("\nZPTTL\r\n"u8)) + { + return RespCommand.ZPTTL; + } break; } break; @@ -1350,10 +1350,6 @@ private RespCommand FastParseArrayCommand(ref int count, ref ReadOnlySpan { return RespCommand.HEXPIRE; } - else if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("ZEXPIRE\r"u8) && *(byte*)(ptr + 12) == '\n') - { - return RespCommand.ZEXPIRE; - } else if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("HINCRBY\r"u8) && *(byte*)(ptr + 12) == '\n') { return RespCommand.HINCRBY; @@ -1406,6 +1402,10 @@ private RespCommand FastParseArrayCommand(ref int count, ref ReadOnlySpan { return RespCommand.ZPOPMIN; } + else if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("ZEXPIRE\r"u8) && *(byte*)(ptr + 12) == '\n') + { + return RespCommand.ZEXPIRE; + } else if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("ZPOPMAX\r"u8) && *(byte*)(ptr + 12) == '\n') { return RespCommand.ZPOPMAX; diff --git a/libs/server/Storage/Session/ObjectStore/SortedSetOps.cs b/libs/server/Storage/Session/ObjectStore/SortedSetOps.cs index 1ea7e32a61..aca7c7867e 100644 --- a/libs/server/Storage/Session/ObjectStore/SortedSetOps.cs +++ b/libs/server/Storage/Session/ObjectStore/SortedSetOps.cs @@ -1235,9 +1235,9 @@ private GarnetStatus SortedSetDifference(ReadOnlySpan } if (pairs == default) - pairs = SortedSetObject.CopyDiff(firstSortedSet.Dictionary, nextSortedSet.Dictionary); + pairs = SortedSetObject.CopyDiff(firstSortedSet, nextSortedSet); else - SortedSetObject.InPlaceDiff(pairs, nextSortedSet.Dictionary); + SortedSetObject.InPlaceDiff(pairs, nextSortedSet); } } @@ -1435,16 +1435,10 @@ private GarnetStatus SortedSetIntersection(ReadOnlySpan(firstSortedSet.Dictionary, ByteArrayComparer.Instance); + pairs = keys.Length == 1 ? firstSortedSet.Dictionary : new Dictionary(firstSortedSet.Dictionary, ByteArrayComparer.Instance); } else { @@ -1455,6 +1449,11 @@ private GarnetStatus SortedSetIntersection(ReadOnlySpan(ReadOnlySpan ("2", "6", "4"), + "ZPEXPIRE" => ("2000", "6000", "4000"), + "ZEXPIREAT" => (DateTimeOffset.UtcNow.AddSeconds(2).ToUnixTimeSeconds().ToString(), DateTimeOffset.UtcNow.AddSeconds(6).ToUnixTimeSeconds().ToString(), DateTimeOffset.UtcNow.AddSeconds(4).ToUnixTimeSeconds().ToString()), + "ZPEXPIREAT" => (DateTimeOffset.UtcNow.AddSeconds(2).ToUnixTimeMilliseconds().ToString(), DateTimeOffset.UtcNow.AddSeconds(6).ToUnixTimeMilliseconds().ToString(), DateTimeOffset.UtcNow.AddSeconds(4).ToUnixTimeMilliseconds().ToString()), + _ => throw new ArgumentException("Invalid command") + }; + + // First set TTL for member1 only + db.Execute(command, "mysortedset", expireTimeMember1, "MEMBERS", "1", "member1"); + db.Execute(command, "mysortedset", expireTimeMember3, "MEMBERS", "1", "member3"); + + // Try setting TTL with option + var result = (RedisResult[])db.Execute(command, "mysortedset", newExpireTimeMember, option, "MEMBERS", "3", "member1", "member2", "member3"); + + switch (option) + { + case "NX": + ClassicAssert.AreEqual(0, (long)result[0]); // member1 has TTL + ClassicAssert.AreEqual(1, (long)result[1]); // member2 no TTL + ClassicAssert.AreEqual(0, (long)result[2]); // member3 has TTL + break; + case "XX": + ClassicAssert.AreEqual(1, (long)result[0]); // member1 has TTL + ClassicAssert.AreEqual(0, (long)result[1]); // member2 no TTL + ClassicAssert.AreEqual(1, (long)result[2]); // member3 has TTL + break; + case "GT": + ClassicAssert.AreEqual(1, (long)result[0]); // 4 > 2 + ClassicAssert.AreEqual(0, (long)result[1]); // no TTL = infinite + ClassicAssert.AreEqual(0, (long)result[2]); // 4 !> 6 + break; + case "LT": + ClassicAssert.AreEqual(0, (long)result[0]); // 4 !< 2 + ClassicAssert.AreEqual(1, (long)result[1]); // no TTL = infinite + ClassicAssert.AreEqual(1, (long)result[2]); // 4 < 6 + break; + } + } + + [Test] + public async Task ZDiffWithExpiredAndExpiringItems() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + // Add items to the sorted sets + db.SortedSetAdd("key1", "a", 1); + db.SortedSetAdd("key1", "b", 2); + db.SortedSetAdd("key1", "c", 3); + db.SortedSetAdd("key1", "d", 4); + db.SortedSetAdd("key1", "e", 5); + + db.SortedSetAdd("key2", "a", 1); + db.SortedSetAdd("key2", "b", 2); + db.SortedSetAdd("key2", "c", 3); + + // Set expiration for some items in key1 + db.Execute("ZPEXPIRE", "key1", "200", "MEMBERS", "2", "a", "c"); + db.Execute("ZPEXPIRE", "key1", "500", "MEMBERS", "1", "d"); + + // Set expiration for matching items in key2 + db.Execute("ZPEXPIRE", "key2", "200", "MEMBERS", "1", "a"); + + await Task.Delay(300); + + // Perform ZDIFF + var diff = db.SortedSetCombine(SetOperation.Difference, ["key1", "key2"]); + ClassicAssert.AreEqual(2, diff.Length); // Only "d" and "e" should remain + + // Perform ZDIFF with scores + var diffWithScores = db.SortedSetCombineWithScores(SetOperation.Difference, ["key1", "key2"]); + ClassicAssert.AreEqual(2, diffWithScores.Length); + ClassicAssert.AreEqual("d", diffWithScores[0].Element.ToString()); + ClassicAssert.AreEqual(4, diffWithScores[0].Score); + ClassicAssert.AreEqual("e", diffWithScores[1].Element.ToString()); + ClassicAssert.AreEqual(5, diffWithScores[1].Score); + + await Task.Delay(300); + + // Perform ZDIFF again after more items have expired + diff = db.SortedSetCombine(SetOperation.Difference, ["key1", "key2"]); + ClassicAssert.AreEqual(1, diff.Length); // Only "e" should remain + + // Perform ZDIFF with scores again + diffWithScores = db.SortedSetCombineWithScores(SetOperation.Difference, ["key1", "key2"]); + ClassicAssert.AreEqual(1, diffWithScores.Length); + ClassicAssert.AreEqual("e", diffWithScores[0].Element.ToString()); + ClassicAssert.AreEqual(5, diffWithScores[0].Score); + } + + [Test] + public async Task ZDiffStoreWithExpiredAndExpiringItems() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + // Add items to the sorted sets + db.SortedSetAdd("key1", "a", 1); + db.SortedSetAdd("key1", "b", 2); + db.SortedSetAdd("key1", "c", 3); + db.SortedSetAdd("key1", "d", 4); + db.SortedSetAdd("key1", "e", 5); + + db.SortedSetAdd("key2", "a", 1); + db.SortedSetAdd("key2", "b", 2); + db.SortedSetAdd("key2", "c", 3); + + // Set expiration for some items in key1 + db.Execute("ZPEXPIRE", "key1", "200", "MEMBERS", "2", "a", "c"); + db.Execute("ZPEXPIRE", "key1", "500", "MEMBERS", "1", "d"); + + // Set expiration for matching items in key2 + db.Execute("ZPEXPIRE", "key2", "200", "MEMBERS", "1", "a"); + + await Task.Delay(300); + + // Perform ZDIFFSTORE + var diffStoreCount = db.SortedSetCombineAndStore(SetOperation.Difference, "key3", ["key1", "key2"]); + ClassicAssert.AreEqual(2, diffStoreCount); // Only "d" and "e" should remain + + // Verify the stored result + var diffStoreResult = db.SortedSetRangeByRankWithScores("key3"); + ClassicAssert.AreEqual(2, diffStoreResult.Length); + ClassicAssert.AreEqual("d", diffStoreResult[0].Element.ToString()); + ClassicAssert.AreEqual(4, diffStoreResult[0].Score); + ClassicAssert.AreEqual("e", diffStoreResult[1].Element.ToString()); + ClassicAssert.AreEqual(5, diffStoreResult[1].Score); + + await Task.Delay(300); + + // Perform ZDIFFSTORE again after more items have expired + diffStoreCount = db.SortedSetCombineAndStore(SetOperation.Difference, "key3", ["key1", "key2"]); + ClassicAssert.AreEqual(1, diffStoreCount); // Only "e" should remain + + // Verify the stored result again + diffStoreResult = db.SortedSetRangeByRankWithScores("key3"); + ClassicAssert.AreEqual(1, diffStoreResult.Length); + ClassicAssert.AreEqual("e", diffStoreResult[0].Element.ToString()); + ClassicAssert.AreEqual(5, diffStoreResult[0].Score); + } + + [Test] + public async Task ZIncrByWithExpiringAndExpiredItems() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + // Add items to the sorted set + db.SortedSetAdd("key1", "a", 1); + db.SortedSetAdd("key1", "b", 2); + db.SortedSetAdd("key1", "c", 3); + + // Set expiration for some items in key1 + db.Execute("ZPEXPIRE", "key1", "200", "MEMBERS", "1", "a"); + + await Task.Delay(10); + + // Try to increment the score of an expiring item + var newScore = db.SortedSetIncrement("key1", "a", 5); + ClassicAssert.AreEqual(6, newScore); + + // Check the TTL of the expiring item + var ttl = db.Execute("ZPTTL", "key1", "MEMBERS", "1", "a"); + ClassicAssert.LessOrEqual((long)ttl, 200); + ClassicAssert.Greater((long)ttl, 0); + + await Task.Delay(200); + + // Check the item has expired + ttl = db.Execute("ZPTTL", "key1", "MEMBERS", "1", "a"); + ClassicAssert.AreEqual(-2, (long)ttl); + + // Try to increment the score of an already expired item + newScore = db.SortedSetIncrement("key1", "a", 5); + ClassicAssert.AreEqual(5, newScore); + + // Verify the item is added back with the new score + var score = db.SortedSetScore("key1", "a"); + ClassicAssert.AreEqual(5, score); + } + + [Test] + public async Task ZInterWithExpiredAndExpiringItems() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + // Add items to the sorted sets + db.SortedSetAdd("key1", "a", 1); + db.SortedSetAdd("key1", "b", 2); + db.SortedSetAdd("key1", "c", 3); + db.SortedSetAdd("key1", "d", 4); + db.SortedSetAdd("key1", "e", 5); + + db.SortedSetAdd("key2", "a", 1); + db.SortedSetAdd("key2", "b", 2); + db.SortedSetAdd("key2", "c", 3); + + db.Execute("ZPEXPIRE", "key1", "200", "MEMBERS", "1", "c"); + db.Execute("ZPEXPIRE", "key1", "500", "MEMBERS", "1", "b"); + db.Execute("ZPEXPIRE", "key2", "200", "MEMBERS", "1", "a"); + + var inter = db.SortedSetCombine(SetOperation.Intersect, ["key1", "key2"]); + ClassicAssert.AreEqual(3, inter.Length); + + await Task.Delay(300); + + var interWithScores = db.SortedSetCombineWithScores(SetOperation.Intersect, ["key1", "key2"]); + ClassicAssert.AreEqual(1, interWithScores.Length); // Only "b" should remain + ClassicAssert.AreEqual("b", interWithScores[0].Element.ToString()); + ClassicAssert.AreEqual(4, interWithScores[0].Score); // Sum of scores + + await Task.Delay(300); + + inter = db.SortedSetCombine(SetOperation.Intersect, ["key1", "key2"]); + ClassicAssert.AreEqual(0, inter.Length); + } + + [Test] + public async Task ZInterCardWithExpiredAndExpiringItems() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + db.SortedSetAdd("key1", "a", 1); + db.SortedSetAdd("key1", "b", 2); + db.SortedSetAdd("key1", "c", 3); + db.SortedSetAdd("key1", "d", 4); + db.SortedSetAdd("key1", "e", 5); + + db.SortedSetAdd("key2", "a", 1); + db.SortedSetAdd("key2", "b", 2); + db.SortedSetAdd("key2", "c", 3); + + db.Execute("ZPEXPIRE", "key1", "200", "MEMBERS", "1", "c"); + db.Execute("ZPEXPIRE", "key1", "500", "MEMBERS", "1", "b"); + db.Execute("ZPEXPIRE", "key2", "200", "MEMBERS", "1", "a"); + + await Task.Delay(300); + + var interCardCount = (long)db.Execute("ZINTERCARD", "2", "key1", "key2"); + ClassicAssert.AreEqual(1, interCardCount); // Only "b" should remain + + await Task.Delay(300); + + interCardCount = (long)db.Execute("ZINTERCARD", "2", "key1", "key2"); + ClassicAssert.AreEqual(0, interCardCount); // No items should remain + } + + [Test] + public async Task ZInterStoreWithExpiredAndExpiringItems() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + db.SortedSetAdd("key1", "a", 1); + db.SortedSetAdd("key1", "b", 2); + db.SortedSetAdd("key1", "c", 3); + db.SortedSetAdd("key1", "d", 4); + db.SortedSetAdd("key1", "e", 5); + + db.SortedSetAdd("key2", "a", 1); + db.SortedSetAdd("key2", "b", 2); + db.SortedSetAdd("key2", "c", 3); + + db.Execute("ZPEXPIRE", "key1", "200", "MEMBERS", "1", "c"); + db.Execute("ZPEXPIRE", "key1", "500", "MEMBERS", "1", "b"); + db.Execute("ZPEXPIRE", "key2", "200", "MEMBERS", "1", "a"); + + await Task.Delay(300); + + var interStoreCount = db.SortedSetCombineAndStore(SetOperation.Intersect, "key3", ["key1", "key2"]); + ClassicAssert.AreEqual(1, interStoreCount); // Only "b" should remain + + var interStoreResult = db.SortedSetRangeByRankWithScores("key3"); + ClassicAssert.AreEqual(1, interStoreResult.Length); + ClassicAssert.AreEqual("b", interStoreResult[0].Element.ToString()); + ClassicAssert.AreEqual(4, interStoreResult[0].Score); // Sum of scores + + await Task.Delay(300); + + interStoreCount = db.SortedSetCombineAndStore(SetOperation.Intersect, "key3", ["key1", "key2"]); + ClassicAssert.AreEqual(0, interStoreCount); // No items should remain + + interStoreResult = db.SortedSetRangeByRankWithScores("key3"); + ClassicAssert.AreEqual(0, interStoreResult.Length); + } + + [Test] + public async Task ZLexCountWithExpiredAndExpiringItems() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + db.SortedSetAdd("key1", "a", 1); + db.SortedSetAdd("key1", "b", 2); + db.SortedSetAdd("key1", "c", 3); + db.SortedSetAdd("key1", "d", 4); + db.SortedSetAdd("key1", "e", 5); + + db.Execute("ZPEXPIRE", "key1", "200", "MEMBERS", "3", "a", "e", "c"); + db.Execute("ZPEXPIRE", "key1", "500", "MEMBERS", "1", "b"); + + await Task.Delay(300); + + var lexCount = (int)db.Execute("ZLEXCOUNT", "key1", "-", "+"); // SortedSetLengthByValue will check - and + to [- and [+ + ClassicAssert.AreEqual(2, lexCount); // Only "b" and "d" should remain + + var lexCountRange = db.SortedSetLengthByValue("key1", "b", "d", Exclude.Stop); + ClassicAssert.AreEqual(1, lexCountRange); // Only "b" should remain within the range + + await Task.Delay(300); + + lexCount = (int)db.Execute("ZLEXCOUNT", "key1", "-", "+"); + ClassicAssert.AreEqual(1, lexCount); // Only "d" should remain + + lexCountRange = db.SortedSetLengthByValue("key1", "b", "d"); + ClassicAssert.AreEqual(1, lexCountRange); // Only "d" should remain within the range + } + + [Test] + public async Task ZMPopWithExpiredItems() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + db.SortedSetAdd("key0", "x", 1); + db.SortedSetAdd("key1", "a", 1); + db.SortedSetAdd("key1", "b", 2); + db.SortedSetAdd("key1", "c", 3); + db.SortedSetAdd("key1", "d", 4); + db.SortedSetAdd("key1", "e", 5); + + // Set expiration for the minimum and maximum items + db.Execute("ZPEXPIRE", "key0", "200", "MEMBERS", "1", "x"); + db.Execute("ZPEXPIRE", "key1", "200", "MEMBERS", "2", "a", "e"); + + await Task.Delay(300); + + // Perform ZMPOP with MIN option + var result = db.Execute("ZMPOP", 2, "key0", "key1", "MIN", "COUNT", 2); + ClassicAssert.IsNotNull(result); + var popResult = (RedisResult[])result; + ClassicAssert.AreEqual("key1", (string)popResult[0]); + + var poppedItems = (RedisResult[])popResult[1]; + ClassicAssert.AreEqual(2, poppedItems.Length); + ClassicAssert.AreEqual("b", (string)poppedItems[0][0]); + ClassicAssert.AreEqual("2", (string)poppedItems[0][1]); + ClassicAssert.AreEqual("c", (string)poppedItems[1][0]); + ClassicAssert.AreEqual("3", (string)poppedItems[1][1]); + + // Perform ZMPOP with MAX option + result = db.Execute("ZMPOP", 2, "key0", "key1", "MAX", "COUNT", 2); + ClassicAssert.IsNotNull(result); + popResult = (RedisResult[])result; + ClassicAssert.AreEqual("key1", (string)popResult[0]); + + poppedItems = (RedisResult[])popResult[1]; + ClassicAssert.AreEqual(1, poppedItems.Length); + ClassicAssert.AreEqual("d", (string)poppedItems[0][0]); + ClassicAssert.AreEqual("4", (string)poppedItems[0][1]); + } + + [Test] + public async Task ZMScoreWithExpiredItems() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + // Add items to the sorted set + db.SortedSetAdd("key1", "a", 1); + db.SortedSetAdd("key1", "b", 2); + db.SortedSetAdd("key1", "c", 3); + db.SortedSetAdd("key1", "d", 4); + db.SortedSetAdd("key1", "e", 5); + + // Set expiration for the minimum and maximum items + db.Execute("ZPEXPIRE", "key1", "200", "MEMBERS", "2", "a", "e"); + + await Task.Delay(300); + + var scores = db.SortedSetScores("key1", ["a", "b", "c", "d", "e"]); + ClassicAssert.AreEqual(5, scores.Length); + ClassicAssert.IsNull(scores[0]); // "a" should be expired + ClassicAssert.AreEqual(2, scores[1]); // "b" should remain + ClassicAssert.AreEqual(3, scores[2]); // "c" should remain + ClassicAssert.AreEqual(4, scores[3]); // "d" should remain + ClassicAssert.IsNull(scores[4]); // "e" should be expired + } + + [Test] + public async Task ZPopMaxWithExpiredItems() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + // Add items to the sorted set + db.SortedSetAdd("key1", "a", 1); + db.SortedSetAdd("key1", "b", 2); + db.SortedSetAdd("key1", "c", 3); + db.SortedSetAdd("key1", "d", 4); + db.SortedSetAdd("key1", "e", 5); + + // Set expiration for the minimum and maximum items + db.Execute("ZPEXPIRE", "key1", "200", "MEMBERS", "2", "c", "e"); + + await Task.Delay(300); + + // Perform ZPOPMAX + var result = db.SortedSetPop("key1", Order.Descending); + ClassicAssert.IsNotNull(result); + ClassicAssert.AreEqual("d", result.Value.Element.ToString()); + ClassicAssert.AreEqual(4, result.Value.Score); + + // Perform ZPOPMAX with COUNT option + var results = db.SortedSetPop("key1", 2, Order.Descending); + ClassicAssert.AreEqual(2, results.Length); + ClassicAssert.AreEqual("b", results[0].Element.ToString()); + ClassicAssert.AreEqual(2, results[0].Score); + ClassicAssert.AreEqual("a", results[1].Element.ToString()); + ClassicAssert.AreEqual(1, results[1].Score); + } + + [Test] + public async Task ZPopMinWithExpiredItems() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + // Add items to the sorted set + db.SortedSetAdd("key1", "a", 1); + db.SortedSetAdd("key1", "b", 2); + db.SortedSetAdd("key1", "c", 3); + db.SortedSetAdd("key1", "d", 4); + db.SortedSetAdd("key1", "e", 5); + + // Set expiration for the minimum and middle items + db.Execute("ZPEXPIRE", "key1", "200", "MEMBERS", "2", "a", "c"); + + await Task.Delay(300); + + // Perform ZPOPMIN + var result = db.SortedSetPop("key1", Order.Ascending); + ClassicAssert.IsNotNull(result); + ClassicAssert.AreEqual("b", result.Value.Element.ToString()); + ClassicAssert.AreEqual(2, result.Value.Score); + + // Perform ZPOPMIN with COUNT option + var results = db.SortedSetPop("key1", 2, Order.Ascending); + ClassicAssert.AreEqual(2, results.Length); + ClassicAssert.AreEqual("d", results[0].Element.ToString()); + ClassicAssert.AreEqual(4, results[0].Score); + ClassicAssert.AreEqual("e", results[1].Element.ToString()); + ClassicAssert.AreEqual(5, results[1].Score); + } + + [Test] + public async Task ZRandMemberWithExpiredItems() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + // Add items to the sorted set + db.SortedSetAdd("key1", "a", 1); + db.SortedSetAdd("key1", "b", 2); + db.SortedSetAdd("key1", "c", 3); + db.SortedSetAdd("key1", "d", 4); + db.SortedSetAdd("key1", "e", 5); + + // Set expiration for some items + db.Execute("ZPEXPIRE", "key1", "200", "MEMBERS", "2", "a", "e"); + + await Task.Delay(300); + + // Perform ZRANDMEMBER + var randMember = db.SortedSetRandomMember("key1"); + + ClassicAssert.IsFalse(randMember.IsNull); + ClassicAssert.IsTrue(new[] { "b", "c", "d" }.Contains(randMember.ToString())); + + // Perform ZRANDMEMBER with count + var randMembers = db.SortedSetRandomMembers("key1", 4); + ClassicAssert.AreEqual(3, randMembers.Length); + CollectionAssert.AreEquivalent(new[] { "b", "c", "d" }, randMembers.Select(member => member.ToString()).ToList()); + + // Perform ZRANDMEMBER with count and WITHSCORES + var randMembersWithScores = db.SortedSetRandomMembersWithScores("key1", 4); + ClassicAssert.AreEqual(3, randMembersWithScores.Length); + CollectionAssert.AreEquivalent(new[] { "b", "c", "d" }, randMembersWithScores.Select(member => member.Element.ToString()).ToList()); + } + + [Test] + public async Task ZRangeWithExpiredItems() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + // Add items to the sorted set + db.SortedSetAdd("key1", "a", 1); + db.SortedSetAdd("key1", "b", 2); + db.SortedSetAdd("key1", "c", 3); + db.SortedSetAdd("key1", "d", 4); + db.SortedSetAdd("key1", "e", 5); + + // Set expiration for some items + db.Execute("ZPEXPIRE", "key1", "200", "MEMBERS", "2", "a", "e"); + + await Task.Delay(300); + + // Perform ZRANGE with BYSCORE option + var result = (RedisValue[])db.Execute("ZRANGE", "key1", "1", "5", "BYSCORE"); + ClassicAssert.AreEqual(3, result.Length); + CollectionAssert.AreEqual(new[] { "b", "c", "d" }, result.Select(r => r.ToString()).ToList()); + + // Perform ZRANGE with BYLEX option + result = (RedisValue[])db.Execute("ZRANGE", "key1", "[b", "[d", "BYLEX"); + ClassicAssert.AreEqual(3, result.Length); + CollectionAssert.AreEqual(new[] { "b", "c", "d" }, result.Select(r => r.ToString()).ToList()); + + // Perform ZRANGE with REV option + result = (RedisValue[])db.Execute("ZRANGE", "key1", "5", "1", "BYSCORE", "REV"); + ClassicAssert.AreEqual(3, result.Length); + CollectionAssert.AreEqual(new[] { "d", "c", "b" }, result.Select(r => r.ToString()).ToList()); + + // Perform ZRANGE with LIMIT option + result = (RedisValue[])db.Execute("ZRANGE", "key1", "1", "5", "BYSCORE", "LIMIT", "1", "2"); + ClassicAssert.AreEqual(2, result.Length); + CollectionAssert.AreEqual(new[] { "c", "d" }, result.Select(r => r.ToString()).ToList()); + } + + [Test] + public async Task ZRangeByLexWithExpiredItems() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + // Add items to the sorted set + db.SortedSetAdd("key1", "a", 1); + db.SortedSetAdd("key1", "b", 1); + db.SortedSetAdd("key1", "c", 1); + db.SortedSetAdd("key1", "d", 1); + db.SortedSetAdd("key1", "e", 1); + + // Set expiration for some items + db.Execute("ZPEXPIRE", "key1", "200", "MEMBERS", "2", "a", "e"); + + await Task.Delay(300); + + // Perform ZRANGEBYLEX with expired items + var result = (RedisResult[])db.Execute("ZRANGEBYLEX", "key1", "[a", "[e"); + ClassicAssert.AreEqual(3, result.Length); + CollectionAssert.AreEqual(new[] { "b", "c", "d" }, result.Select(r => r.ToString()).ToList()); + } + + [Test] + public async Task ZRangeByScoreWithExpiredItems() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + // Add items to the sorted set + db.SortedSetAdd("key1", "a", 1); + db.SortedSetAdd("key1", "b", 2); + db.SortedSetAdd("key1", "c", 3); + db.SortedSetAdd("key1", "d", 4); + db.SortedSetAdd("key1", "e", 5); + + // Set expiration for some items + db.Execute("ZPEXPIRE", "key1", "200", "MEMBERS", "2", "a", "e"); + + await Task.Delay(300); + + // Perform ZRANGEBYSCORE + var result = (RedisValue[])db.Execute("ZRANGEBYSCORE", "key1", "1", "5"); + ClassicAssert.AreEqual(3, result.Length); + CollectionAssert.AreEqual(new[] { "b", "c", "d" }, result.Select(r => r.ToString()).ToList()); + + // Perform ZRANGEBYSCORE with expired items + result = (RedisValue[])db.Execute("ZRANGEBYSCORE", "key1", "1", "5"); + ClassicAssert.AreEqual(3, result.Length); + CollectionAssert.AreEqual(new[] { "b", "c", "d" }, result.Select(r => r.ToString()).ToList()); + } + + [Test] + public async Task ZRangeStoreWithExpiredItems() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + // Add items to the sorted set + db.SortedSetAdd("key1", "a", 1); + db.SortedSetAdd("key1", "b", 2); + db.SortedSetAdd("key1", "c", 3); + db.SortedSetAdd("key1", "d", 4); + db.SortedSetAdd("key1", "e", 5); + + // Set expiration for some items + db.Execute("ZPEXPIRE", "key1", "200", "MEMBERS", "2", "a", "e"); + + await Task.Delay(300); + + // Perform ZRANGESTORE with BYSCORE option + db.Execute("ZRANGESTORE", "key2", "key1", "1", "5", "BYSCORE"); + var result = db.SortedSetRangeByRank("key2"); + ClassicAssert.AreEqual(3, result.Length); + CollectionAssert.AreEqual(new[] { "b", "c", "d" }, result.Select(r => r.ToString()).ToList()); + + // Perform ZRANGESTORE with BYLEX option + db.Execute("ZRANGESTORE", "key2", "key1", "[b", "[d", "BYLEX"); + result = db.SortedSetRangeByRank("key2"); + ClassicAssert.AreEqual(3, result.Length); + CollectionAssert.AreEqual(new[] { "b", "c", "d" }, result.Select(r => r.ToString()).ToList()); + + // Perform ZRANGESTORE with REV option + db.Execute("ZRANGESTORE", "key2", "key1", "5", "1", "BYSCORE", "REV"); + result = db.SortedSetRangeByRank("key2"); + ClassicAssert.AreEqual(3, result.Length); + CollectionAssert.AreEqual(new[] { "b", "c", "d" }, result.Select(r => r.ToString()).ToList()); + + // Perform ZRANGESTORE with LIMIT option + db.Execute("ZRANGESTORE", "key2", "key1", "1", "5", "BYSCORE", "LIMIT", "1", "2"); + result = db.SortedSetRangeByRank("key2"); + ClassicAssert.AreEqual(2, result.Length); + CollectionAssert.AreEqual(new[] { "c", "d" }, result.Select(r => r.ToString()).ToList()); + } + + [Test] + public async Task ZRankWithExpiredItems() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + // Add items to the sorted set + db.SortedSetAdd("key1", "a", 1); + db.SortedSetAdd("key1", "b", 2); + db.SortedSetAdd("key1", "c", 3); + db.SortedSetAdd("key1", "d", 4); + db.SortedSetAdd("key1", "e", 5); + + // Set expiration for some items + db.Execute("ZPEXPIRE", "key1", "200", "MEMBERS", "2", "a", "e"); + + await Task.Delay(300); + + // Perform ZRANK + var rank = db.SortedSetRank("key1", "a"); + ClassicAssert.IsNull(rank); // "a" should be expired + + rank = db.SortedSetRank("key1", "b"); + ClassicAssert.AreEqual(0, rank); // "b" should be at rank 0 + } + + [Test] + public async Task ZRemWithExpiredItems() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + // Add items to the sorted set + db.SortedSetAdd("key1", "a", 1); + db.SortedSetAdd("key1", "b", 2); + db.SortedSetAdd("key1", "c", 3); + db.SortedSetAdd("key1", "d", 4); + db.SortedSetAdd("key1", "e", 5); + + // Set expiration for some items + db.Execute("ZPEXPIRE", "key1", "200", "MEMBERS", "2", "a", "e"); + + await Task.Delay(300); + + // Perform ZREM on expired and non-expired items + var removedCount = db.SortedSetRemove("key1", ["a", "b", "e"]); + ClassicAssert.AreEqual(1, removedCount); // "a" and "e" should be expired, "b" should be removed + + // Verify remaining items in the sorted set + var remainingItems = db.SortedSetRangeByRank("key1"); + ClassicAssert.AreEqual(2, remainingItems.Length); + CollectionAssert.AreEqual(new[] { "c", "d" }, remainingItems.Select(r => r.ToString()).ToList()); + } + + [Test] + public async Task ZRemRangeByLexWithExpiredItems() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + // Add items to the sorted set + db.SortedSetAdd("key1", "a", 1); + db.SortedSetAdd("key1", "b", 1); + db.SortedSetAdd("key1", "c", 1); + db.SortedSetAdd("key1", "d", 1); + db.SortedSetAdd("key1", "e", 1); + + // Set expiration for some items + db.Execute("ZPEXPIRE", "key1", "200", "MEMBERS", "2", "a", "e"); + + await Task.Delay(300); + + // Perform ZREMRANGEBYLEX with expired items + var removedCount = db.Execute("ZREMRANGEBYLEX", "key1", "[a", "[e"); + ClassicAssert.AreEqual(3, (int)removedCount); // Only "b", "c", and "d" should be removed + + // Verify remaining items in the sorted set + var remainingItems = db.SortedSetRangeByRank("key1"); + ClassicAssert.AreEqual(0, remainingItems.Length); // All items should be removed + } + + [Test] + public async Task ZRemRangeByRankWithExpiredItems() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + // Add items to the sorted set + db.SortedSetAdd("key1", "a", 1); + db.SortedSetAdd("key1", "b", 2); + db.SortedSetAdd("key1", "c", 3); + db.SortedSetAdd("key1", "d", 4); + db.SortedSetAdd("key1", "e", 5); + + // Set expiration for some items + db.Execute("ZPEXPIRE", "key1", "200", "MEMBERS", "2", "a", "e"); + + await Task.Delay(300); + + // Perform ZREMRANGEBYRANK with expired items + var removedCount = db.Execute("ZREMRANGEBYRANK", "key1", 0, 1); + ClassicAssert.AreEqual(2, (int)removedCount); // Only "b" and "c" should be removed + + // Verify remaining items in the sorted set + var remainingItems = db.SortedSetRangeByRank("key1"); + ClassicAssert.AreEqual(1, remainingItems.Length); + CollectionAssert.AreEqual(new[] { "d" }, remainingItems.Select(r => r.ToString()).ToList()); + } + + [Test] + public async Task ZRemRangeByScoreWithExpiredItems() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + // Add items to the sorted set + db.SortedSetAdd("key1", "a", 1); + db.SortedSetAdd("key1", "b", 2); + db.SortedSetAdd("key1", "c", 3); + db.SortedSetAdd("key1", "d", 4); + db.SortedSetAdd("key1", "e", 5); + + // Set expiration for some items + db.Execute("ZPEXPIRE", "key1", "200", "MEMBERS", "2", "a", "e"); + + await Task.Delay(300); + + // Perform ZREMRANGEBYSCORE with expired items + var removedCount = db.Execute("ZREMRANGEBYSCORE", "key1", 1, 5); + ClassicAssert.AreEqual(3, (int)removedCount); // Only "b", "c", and "d" should be removed + + // Verify remaining items in the sorted set + var remainingItems = db.SortedSetRangeByRank("key1"); + ClassicAssert.AreEqual(0, remainingItems.Length); // All items should be removed + } + + [Test] + public async Task ZRevRangeWithExpiredItems() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + // Add items to the sorted set + db.SortedSetAdd("key1", "a", 1); + db.SortedSetAdd("key1", "b", 2); + db.SortedSetAdd("key1", "c", 3); + db.SortedSetAdd("key1", "d", 4); + db.SortedSetAdd("key1", "e", 5); + + // Set expiration for some items + db.Execute("ZPEXPIRE", "key1", "200", "MEMBERS", "2", "a", "e"); + + await Task.Delay(300); + + // Perform ZREVRANGE with expired items + var result = db.Execute("ZREVRANGE", "key1", 0, -1); + var items = (RedisValue[])result; + ClassicAssert.AreEqual(3, items.Length); // Only "b", "c", and "d" should remain + CollectionAssert.AreEqual(new[] { "d", "c", "b" }, items.Select(r => r.ToString()).ToList()); + } + + [Test] + public async Task ZRevRangeByLexWithExpiredItems() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + // Add items to the sorted set + db.SortedSetAdd("key1", "a", 1); + db.SortedSetAdd("key1", "b", 1); + db.SortedSetAdd("key1", "c", 1); + db.SortedSetAdd("key1", "d", 1); + db.SortedSetAdd("key1", "e", 1); + + // Set expiration for some items + db.Execute("ZPEXPIRE", "key1", "200", "MEMBERS", "2", "a", "e"); + + await Task.Delay(300); + + // Perform ZREVRANGEBYLEX with expired items + var result = db.Execute("ZREVRANGEBYLEX", "key1", "[e", "[a"); + var items = (RedisValue[])result; + ClassicAssert.AreEqual(3, items.Length); // Only "b", "c", and "d" should remain + CollectionAssert.AreEqual(new[] { "d", "c", "b" }, items.Select(r => r.ToString()).ToList()); + } + + [Test] + public async Task ZRevRangeByScoreWithExpiredItems() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + // Add items to the sorted set + db.SortedSetAdd("key1", "a", 1); + db.SortedSetAdd("key1", "b", 2); + db.SortedSetAdd("key1", "c", 3); + db.SortedSetAdd("key1", "d", 4); + db.SortedSetAdd("key1", "e", 5); + + // Set expiration for some items + db.Execute("ZPEXPIRE", "key1", "200", "MEMBERS", "2", "a", "e"); + + await Task.Delay(300); + + // Perform ZREVRANGEBYSCORE with expired items + var result = db.Execute("ZREVRANGEBYSCORE", "key1", 5, 1); + var items = (RedisValue[])result; + ClassicAssert.AreEqual(3, items.Length); // Only "b", "c", and "d" should remain + CollectionAssert.AreEqual(new[] { "d", "c", "b" }, items.Select(r => r.ToString()).ToList()); + } + + [Test] + public async Task ZRevRankWithExpiredItems() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + // Add items to the sorted set + db.SortedSetAdd("key1", "a", 1); + db.SortedSetAdd("key1", "b", 2); + db.SortedSetAdd("key1", "c", 3); + db.SortedSetAdd("key1", "d", 4); + db.SortedSetAdd("key1", "e", 5); + + // Set expiration for some items + db.Execute("ZPEXPIRE", "key1", "200", "MEMBERS", "2", "a", "e"); + + await Task.Delay(300); + + // Perform ZREVRANK on expired and non-expired items + var result = db.Execute("ZREVRANK", "key1", "a"); + ClassicAssert.True(result.IsNull); // "a" should be expired + + result = db.Execute("ZREVRANK", "key1", "b"); + ClassicAssert.AreEqual(2, (int)result); // "b" should be at reverse rank 2 + } + + [Test] + public async Task ZScanWithExpiredItems() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + // Add items to the sorted set + db.SortedSetAdd("key1", "a", 1); + db.SortedSetAdd("key1", "b", 2); + db.SortedSetAdd("key1", "c", 3); + db.SortedSetAdd("key1", "d", 4); + db.SortedSetAdd("key1", "e", 5); + + // Set expiration for some items + db.Execute("ZPEXPIRE", "key1", "200", "MEMBERS", "2", "a", "e"); + db.Execute("ZPEXPIRE", "key1", "1000", "MEMBERS", "1", "c"); + + await Task.Delay(300); + + // Perform ZSCAN + var result = db.Execute("ZSCAN", "key1", "0"); + var items = (RedisResult[])result; + var cursor = (long)items[0]; + var elements = (RedisValue[])items[1]; + + // Verify that expired items are not returned + ClassicAssert.AreEqual(6, elements.Length); // Only "b", "c", and "d" should remain + CollectionAssert.AreEqual(new[] { "b", "2", "c", "3", "d", "4" }, elements.Select(r => r.ToString()).ToList()); + ClassicAssert.AreEqual(0, cursor); // Ensure the cursor indicates the end of the collection + } + + [Test] + public async Task ZScoreWithExpiringAndExpiredItems() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + // Add items to the sorted set + db.SortedSetAdd("key1", "a", 1); + db.SortedSetAdd("key1", "b", 2); + db.SortedSetAdd("key1", "c", 3); + + // Set expiration for some items in key1 + db.Execute("ZPEXPIRE", "key1", "200", "MEMBERS", "1", "a"); + + await Task.Delay(10); + + // Check the score of an expiring item + var score = db.SortedSetScore("key1", "a"); + ClassicAssert.AreEqual(1, score); + + // Check the TTL of the expiring item + var ttl = db.Execute("ZPTTL", "key1", "MEMBERS", "1", "a"); + ClassicAssert.LessOrEqual((long)ttl, 200); + ClassicAssert.Greater((long)ttl, 0); + + await Task.Delay(200); + + // Check the item has expired + ttl = db.Execute("ZPTTL", "key1", "MEMBERS", "1", "a"); + ClassicAssert.AreEqual(-2, (long)ttl); + + // Check the score of an already expired item + score = db.SortedSetScore("key1", "a"); + ClassicAssert.IsNull(score); + + // Check the score of a non-expiring item + score = db.SortedSetScore("key1", "b"); + ClassicAssert.AreEqual(2, score); + } + + [Test] + public async Task ZUnionWithExpiredItems() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + // Add items to the sorted sets + db.SortedSetAdd("key1", "a", 1); + db.SortedSetAdd("key1", "b", 2); + db.SortedSetAdd("key1", "c", 3); + db.SortedSetAdd("key1", "d", 4); + db.SortedSetAdd("key1", "e", 5); + + db.SortedSetAdd("key2", "a", 1); + db.SortedSetAdd("key2", "b", 2); + db.SortedSetAdd("key2", "c", 3); + + db.Execute("ZPEXPIRE", "key1", "200", "MEMBERS", "2", "a", "c"); + db.Execute("ZPEXPIRE", "key1", "500", "MEMBERS", "1", "b"); + db.Execute("ZPEXPIRE", "key2", "200", "MEMBERS", "1", "a"); + + var union = db.SortedSetCombine(SetOperation.Union, [ "key1", "key2" ]); + ClassicAssert.AreEqual(5, union.Length); + + await Task.Delay(300); + + var unionWithScores = db.SortedSetCombineWithScores(SetOperation.Union, [ "key1", "key2" ]); + ClassicAssert.AreEqual(4, unionWithScores.Length); + } + + [Test] + public async Task ZUnionStoreWithExpiredItems() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + // Add items to the sorted sets + db.SortedSetAdd("key1", "a", 1); + db.SortedSetAdd("key1", "b", 2); + db.SortedSetAdd("key1", "c", 3); + db.SortedSetAdd("key1", "d", 4); + db.SortedSetAdd("key1", "e", 5); + + db.SortedSetAdd("key2", "a", 1); + db.SortedSetAdd("key2", "b", 2); + db.SortedSetAdd("key2", "c", 3); + + db.Execute("ZPEXPIRE", "key1", "200", "MEMBERS", "2", "a", "c"); + db.Execute("ZPEXPIRE", "key1", "1000", "MEMBERS", "1", "b"); + db.Execute("ZPEXPIRE", "key2", "200", "MEMBERS", "1", "a"); + + var unionStoreCount = db.SortedSetCombineAndStore(SetOperation.Union, "key3", ["key1", "key2"]); + ClassicAssert.AreEqual(5, unionStoreCount); + var unionStoreResult = db.SortedSetRangeByRankWithScores("key3"); + ClassicAssert.AreEqual(5, unionStoreResult.Length); + + await Task.Delay(300); + + unionStoreCount = db.SortedSetCombineAndStore(SetOperation.Union, "key3", ["key1", "key2"]); + ClassicAssert.AreEqual(4, unionStoreCount); + unionStoreResult = db.SortedSetRangeByRankWithScores("key3"); + ClassicAssert.AreEqual(4, unionStoreResult.Length); + CollectionAssert.AreEquivalent(new[] { "b", "c", "d", "e" }, unionStoreResult.Select(x => x.Element.ToString())); + } + #endregion #region LightClientTests From 83ec38c66c9a3e6330e7c4b9f2b7caee96cc1560 Mon Sep 17 00:00:00 2001 From: Vijay-Nirmal Date: Mon, 10 Feb 2025 02:19:26 +0530 Subject: [PATCH 4/9] Format fix --- libs/server/Objects/SortedSet/SortedSetObject.cs | 6 +++--- test/Garnet.test/RespSortedSetTests.cs | 10 +++++----- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/libs/server/Objects/SortedSet/SortedSetObject.cs b/libs/server/Objects/SortedSet/SortedSetObject.cs index 48cf6cb809..de764323f1 100644 --- a/libs/server/Objects/SortedSet/SortedSetObject.cs +++ b/libs/server/Objects/SortedSet/SortedSetObject.cs @@ -457,7 +457,7 @@ public override unsafe void Scan(long start, out List items, out long cu { items.Add(item.Key); addToList = true; - } + } else { fixed (byte* keyPtr = item.Key) @@ -466,9 +466,9 @@ public override unsafe void Scan(long start, out List items, out long cu { items.Add(item.Key); addToList = true; - } } } + } if (addToList) { @@ -696,7 +696,7 @@ private int Persist(byte[] key) return -2; } - return TryRemoveExpiration(key) ? 1 : - 1; + return TryRemoveExpiration(key) ? 1 : -1; } [MethodImpl(MethodImplOptions.AggressiveInlining)] diff --git a/test/Garnet.test/RespSortedSetTests.cs b/test/Garnet.test/RespSortedSetTests.cs index 9803d660c2..783830eadb 100644 --- a/test/Garnet.test/RespSortedSetTests.cs +++ b/test/Garnet.test/RespSortedSetTests.cs @@ -20,11 +20,11 @@ namespace Garnet.test { using TestBasicGarnetApi = GarnetApi, - SpanByteAllocator>>, - BasicContext>, - GenericAllocator>>>>; + /* MainStoreFunctions */ StoreFunctions, + SpanByteAllocator>>, + BasicContext>, + GenericAllocator>>>>; [TestFixture] public class RespSortedSetTests From c23682827457531f18e303685104d39dee9ab84a Mon Sep 17 00:00:00 2001 From: Vijay-Nirmal Date: Mon, 10 Feb 2025 02:23:53 +0530 Subject: [PATCH 5/9] Fixed formating --- test/Garnet.test/RespSortedSetTests.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/Garnet.test/RespSortedSetTests.cs b/test/Garnet.test/RespSortedSetTests.cs index 783830eadb..1bedf6e67c 100644 --- a/test/Garnet.test/RespSortedSetTests.cs +++ b/test/Garnet.test/RespSortedSetTests.cs @@ -3075,12 +3075,12 @@ public async Task ZUnionWithExpiredItems() db.Execute("ZPEXPIRE", "key1", "500", "MEMBERS", "1", "b"); db.Execute("ZPEXPIRE", "key2", "200", "MEMBERS", "1", "a"); - var union = db.SortedSetCombine(SetOperation.Union, [ "key1", "key2" ]); + var union = db.SortedSetCombine(SetOperation.Union, ["key1", "key2"]); ClassicAssert.AreEqual(5, union.Length); await Task.Delay(300); - var unionWithScores = db.SortedSetCombineWithScores(SetOperation.Union, [ "key1", "key2" ]); + var unionWithScores = db.SortedSetCombineWithScores(SetOperation.Union, ["key1", "key2"]); ClassicAssert.AreEqual(4, unionWithScores.Length); } From 824ac79ee9d4c346f9dc652df5aa4113f933cc3a Mon Sep 17 00:00:00 2001 From: Vijay-Nirmal Date: Fri, 14 Feb 2025 13:48:24 +0530 Subject: [PATCH 6/9] Fixed review commands --- libs/host/Configuration/Options.cs | 10 +- libs/host/defaults.conf | 7 +- libs/server/Servers/GarnetServerOptions.cs | 9 +- libs/server/StoreWrapper.cs | 53 +--- .../GarnetCommandsInfo.json | 250 ++++++++++++++++++ website/docs/commands/garnet-specific.md | 235 ++++++++++++++++ website/docs/getting-started/configuration.md | 3 +- 7 files changed, 501 insertions(+), 66 deletions(-) diff --git a/libs/host/Configuration/Options.cs b/libs/host/Configuration/Options.cs index 71752a57e3..594623e96d 100644 --- a/libs/host/Configuration/Options.cs +++ b/libs/host/Configuration/Options.cs @@ -235,13 +235,10 @@ internal sealed class Options [IntRangeValidation(0, int.MaxValue)] [Option("compaction-freq", Required = false, HelpText = "Background hybrid log compaction frequency in seconds. 0 = disabled (compaction performed before checkpointing instead)")] public int CompactionFrequencySecs { get; set; } - [IntRangeValidation(0, int.MaxValue)] - [Option("hcollect-freq", Required = false, HelpText = "Frequency in seconds for the background task to perform Hash collection. 0 = disabled. Hash collect is used to delete expired fields from hash without waiting for a write operation. Use the HCOLLECT API to collect on-demand.")] - public int HashCollectFrequencySecs { get; set; } [IntRangeValidation(0, int.MaxValue)] - [Option("zcollect-freq", Required = false, HelpText = "Frequency in seconds for the background task to perform Sorted Set collection. 0 = disabled. Sorted Set collect is used to delete expired members from Sorted Set without waiting for a write operation. Use the ZCOLLECT API to collect on-demand.")] - public int SortedSetCollectFrequencySecs { get; set; } + [Option("expired-object-collection-freq", Required = false, HelpText = "Frequency in seconds for the background task to perform object collection which removes expired members within object from memory. 0 = disabled. Use the HCOLLECT and ZCOLLECT API to collect on-demand.")] + public int ExpiredObjectCollectionFrequencySecs { get; set; } [Option("compaction-type", Required = false, HelpText = "Hybrid log compaction type. Value options: None - no compaction, Shift - shift begin address without compaction (data loss), Scan - scan old pages and move live records to tail (no data loss), Lookup - lookup each record in compaction range, for record liveness checking using hash chain (no data loss)")] public LogCompactionType CompactionType { get; set; } @@ -729,8 +726,7 @@ public GarnetServerOptions GetServerOptions(ILogger logger = null) WaitForCommit = WaitForCommit.GetValueOrDefault(), AofSizeLimit = AofSizeLimit, CompactionFrequencySecs = CompactionFrequencySecs, - HashCollectFrequencySecs = HashCollectFrequencySecs, - SortedSetCollectFrequencySecs = SortedSetCollectFrequencySecs, + ExpiredObjectCollectionFrequencySecs = ExpiredObjectCollectionFrequencySecs, CompactionType = CompactionType, CompactionForceDelete = CompactionForceDelete.GetValueOrDefault(), CompactionMaxSegments = CompactionMaxSegments, diff --git a/libs/host/defaults.conf b/libs/host/defaults.conf index 186384052a..a243dd93d3 100644 --- a/libs/host/defaults.conf +++ b/libs/host/defaults.conf @@ -162,11 +162,8 @@ /* Background hybrid log compaction frequency in seconds. 0 = disabled (compaction performed before checkpointing instead) */ "CompactionFrequencySecs" : 0, - /* Frequency in seconds for the background task to perform Hash collection. 0 = disabled. Hash collect is used to delete expired fields from hash without waiting for a write operation. Use the HCOLLECT API to collect on-demand. */ - "HashCollectFrequencySecs" : 0, - - /* Frequency in seconds for the background task to perform Sorted Set collection. 0 = disabled. Sorted Set collect is used to delete expired members from Sorted Set without waiting for a write operation. Use the HCOLLECT API to collect on-demand. */ - "SortedSetCollectFrequencySecs" : 0, + /* Frequency in seconds for the background task to perform object collection which removes expired members within object from memory. 0 = disabled. Use the HCOLLECT and ZCOLLECT API to collect on-demand. */ + "ExpiredObjectCollectionFrequencySecs" : 0, /* Hybrid log compaction type. Value options: */ /* None - no compaction */ diff --git a/libs/server/Servers/GarnetServerOptions.cs b/libs/server/Servers/GarnetServerOptions.cs index 2bcc007204..283b478f94 100644 --- a/libs/server/Servers/GarnetServerOptions.cs +++ b/libs/server/Servers/GarnetServerOptions.cs @@ -137,14 +137,9 @@ public class GarnetServerOptions : ServerOptions public int CompactionFrequencySecs = 0; /// - /// Hash collection frequency in seconds. 0 = disabled. Hash collect is used to delete expired fields from hash without waiting for a write operation. + /// Frequency in seconds for the background task to perform object collection which removes expired members within object from memory. 0 = disabled. Use the HCOLLECT and ZCOLLECT API to collect on-demand. /// - public int HashCollectFrequencySecs = 0; - - /// - /// Hash collection frequency in seconds. 0 = disabled. Sorted Set collect is used to delete expired members from Sorted Set without waiting for a write operation. - /// - public int SortedSetCollectFrequencySecs = 0; + public int ExpiredObjectCollectionFrequencySecs = 0; /// /// Hybrid log compaction type. diff --git a/libs/server/StoreWrapper.cs b/libs/server/StoreWrapper.cs index 6c6edf77ac..2eae502f22 100644 --- a/libs/server/StoreWrapper.cs +++ b/libs/server/StoreWrapper.cs @@ -428,9 +428,9 @@ async Task CompactionTask(int compactionFrequencySecs, CancellationToken token = } } - async Task HashCollectTask(int hashCollectFrequencySecs, CancellationToken token = default) + async Task ObjectCollectTask(int objectCollectFrequencySecs, CancellationToken token = default) { - Debug.Assert(hashCollectFrequencySecs > 0); + Debug.Assert(objectCollectFrequencySecs > 0); try { var scratchBufferManager = new ScratchBufferManager(); @@ -438,7 +438,7 @@ async Task HashCollectTask(int hashCollectFrequencySecs, CancellationToken token if (objectStore is null) { - logger?.LogWarning("HashCollectFrequencySecs option is configured but Object store is disabled. Stopping the background hash collect task."); + logger?.LogWarning("ExpiredObjectCollectionFrequencySecs option is configured but Object store is disabled. Stopping the background hash collect task."); return; } @@ -447,8 +447,9 @@ async Task HashCollectTask(int hashCollectFrequencySecs, CancellationToken token if (token.IsCancellationRequested) return; ExecuteHashCollect(scratchBufferManager, storageSession); + ExecuteSortedSetCollect(scratchBufferManager, storageSession); - await Task.Delay(TimeSpan.FromSeconds(hashCollectFrequencySecs), token); + await Task.Delay(TimeSpan.FromSeconds(objectCollectFrequencySecs), token); } } catch (TaskCanceledException) when (token.IsCancellationRequested) @@ -457,7 +458,7 @@ async Task HashCollectTask(int hashCollectFrequencySecs, CancellationToken token } catch (Exception ex) { - logger?.LogCritical(ex, "Unknown exception received for background hash collect task. Hash collect task won't be resumed."); + logger?.LogCritical(ex, "Unknown exception received for background hash collect task. Object collect task won't be resumed."); } static void ExecuteHashCollect(ScratchBufferManager scratchBufferManager, StorageSession storageSession) @@ -469,39 +470,6 @@ static void ExecuteHashCollect(ScratchBufferManager scratchBufferManager, Storag storageSession.HashCollect(key, ref input, ref storageSession.objectStoreBasicContext); scratchBufferManager.Reset(); } - } - - async Task SortedSetCollectTask(int sortedSetCollectFrequencySecs, CancellationToken token = default) - { - Debug.Assert(sortedSetCollectFrequencySecs > 0); - try - { - var scratchBufferManager = new ScratchBufferManager(); - using var storageSession = new StorageSession(this, scratchBufferManager, null, null, logger); - - if (objectStore is null) - { - logger?.LogWarning("SortedSetCollectFrequencySecs option is configured but Object store is disabled. Stopping the background sortedSet collect task."); - return; - } - - while (true) - { - if (token.IsCancellationRequested) return; - - ExecuteSortedSetCollect(scratchBufferManager, storageSession); - - await Task.Delay(TimeSpan.FromSeconds(sortedSetCollectFrequencySecs), token); - } - } - catch (TaskCanceledException) when (token.IsCancellationRequested) - { - // Suppress the exception if the task was cancelled because of store wrapper disposal - } - catch (Exception ex) - { - logger?.LogCritical(ex, "Unknown exception received for background sortedSet collect task. SortedSet collect task won't be resumed."); - } static void ExecuteSortedSetCollect(ScratchBufferManager scratchBufferManager, StorageSession storageSession) { @@ -668,14 +636,9 @@ internal void Start() Task.Run(async () => await CompactionTask(serverOptions.CompactionFrequencySecs, ctsCommit.Token)); } - if (serverOptions.HashCollectFrequencySecs > 0) - { - Task.Run(async () => await HashCollectTask(serverOptions.HashCollectFrequencySecs, ctsCommit.Token)); - } - - if (serverOptions.SortedSetCollectFrequencySecs > 0) + if (serverOptions.ExpiredObjectCollectionFrequencySecs > 0) { - Task.Run(async () => await SortedSetCollectTask(serverOptions.SortedSetCollectFrequencySecs, ctsCommit.Token)); + Task.Run(async () => await ObjectCollectTask(serverOptions.ExpiredObjectCollectionFrequencySecs, ctsCommit.Token)); } if (serverOptions.AdjustedIndexMaxCacheLines > 0 || serverOptions.AdjustedObjectStoreIndexMaxCacheLines > 0) diff --git a/playground/CommandInfoUpdater/GarnetCommandsInfo.json b/playground/CommandInfoUpdater/GarnetCommandsInfo.json index 04570affc7..e26b8771b0 100644 --- a/playground/CommandInfoUpdater/GarnetCommandsInfo.json +++ b/playground/CommandInfoUpdater/GarnetCommandsInfo.json @@ -754,5 +754,255 @@ "Flags": "RO" } ] + }, + { + "Command": "ZCOLLECT", + "Name": "ZCOLLECT", + "Arity": 2, + "Flags": "Admin, Write", + "FirstKey": 1, + "LastKey": 1, + "Step": 1, + "AclCategories": "SortedSet, Write, Admin, Garnet", + "KeySpecifications": [ + { + "BeginSearch": { + "TypeDiscriminator": "BeginSearchIndex", + "Index": 1 + }, + "FindKeys": { + "TypeDiscriminator": "FindKeysRange", + "LastKey": 0, + "KeyStep": 1, + "Limit": 0 + }, + "Flags": "RW, Access, Update" + } + ] + }, + { + "Command": "ZEXPIRE", + "Name": "ZEXPIRE", + "Arity": -6, + "Flags": "DenyOom, Fast, Write", + "FirstKey": 1, + "LastKey": 1, + "Step": 1, + "AclCategories": "SortedSet, Fast, Write, Garnet", + "KeySpecifications": [ + { + "BeginSearch": { + "TypeDiscriminator": "BeginSearchIndex", + "Index": 1 + }, + "FindKeys": { + "TypeDiscriminator": "FindKeysRange", + "LastKey": 0, + "KeyStep": 1, + "Limit": 0 + }, + "Flags": "RW, Update" + } + ] + }, + { + "Command": "ZEXPIREAT", + "Name": "ZEXPIREAT", + "Arity": -6, + "Flags": "DenyOom, Fast, Write", + "FirstKey": 1, + "LastKey": 1, + "Step": 1, + "AclCategories": "SortedSet, Fast, Write, Garnet", + "KeySpecifications": [ + { + "BeginSearch": { + "TypeDiscriminator": "BeginSearchIndex", + "Index": 1 + }, + "FindKeys": { + "TypeDiscriminator": "FindKeysRange", + "LastKey": 0, + "KeyStep": 1, + "Limit": 0 + }, + "Flags": "RW, Update" + } + ] + }, + { + "Command": "ZEXPIRETIME", + "Name": "ZEXPIRETIME", + "Arity": -5, + "Flags": "Fast, ReadOnly", + "FirstKey": 1, + "LastKey": 1, + "Step": 1, + "AclCategories": "SortedSet, Fast, Read, Garnet", + "KeySpecifications": [ + { + "BeginSearch": { + "TypeDiscriminator": "BeginSearchIndex", + "Index": 1 + }, + "FindKeys": { + "TypeDiscriminator": "FindKeysRange", + "LastKey": 0, + "KeyStep": 1, + "Limit": 0 + }, + "Flags": "RO, Access" + } + ] + }, + { + "Command": "ZPERSIST", + "Name": "ZPERSIST", + "Arity": -5, + "Flags": "Fast, Write", + "FirstKey": 1, + "LastKey": 1, + "Step": 1, + "AclCategories": "SortedSet, Fast, Write, Garnet", + "KeySpecifications": [ + { + "BeginSearch": { + "TypeDiscriminator": "BeginSearchIndex", + "Index": 1 + }, + "FindKeys": { + "TypeDiscriminator": "FindKeysRange", + "LastKey": 0, + "KeyStep": 1, + "Limit": 0 + }, + "Flags": "RW, Update" + } + ] + }, + { + "Command": "ZPEXPIRE", + "Name": "ZPEXPIRE", + "Arity": -6, + "Flags": "DenyOom, Fast, Write", + "FirstKey": 1, + "LastKey": 1, + "Step": 1, + "AclCategories": "SortedSet, Fast, Write, Garnet", + "KeySpecifications": [ + { + "BeginSearch": { + "TypeDiscriminator": "BeginSearchIndex", + "Index": 1 + }, + "FindKeys": { + "TypeDiscriminator": "FindKeysRange", + "LastKey": 0, + "KeyStep": 1, + "Limit": 0 + }, + "Flags": "RW, Update" + } + ] + }, + { + "Command": "ZPEXPIREAT", + "Name": "ZPEXPIREAT", + "Arity": -6, + "Flags": "DenyOom, Fast, Write", + "FirstKey": 1, + "LastKey": 1, + "Step": 1, + "AclCategories": "SortedSet, Fast, Write, Garnet", + "KeySpecifications": [ + { + "BeginSearch": { + "TypeDiscriminator": "BeginSearchIndex", + "Index": 1 + }, + "FindKeys": { + "TypeDiscriminator": "FindKeysRange", + "LastKey": 0, + "KeyStep": 1, + "Limit": 0 + }, + "Flags": "RW, Update" + } + ] + }, + { + "Command": "ZPEXPIRETIME", + "Name": "ZPEXPIRETIME", + "Arity": -5, + "Flags": "Fast, ReadOnly", + "FirstKey": 1, + "LastKey": 1, + "Step": 1, + "AclCategories": "SortedSet, Fast, Read, Garnet", + "KeySpecifications": [ + { + "BeginSearch": { + "TypeDiscriminator": "BeginSearchIndex", + "Index": 1 + }, + "FindKeys": { + "TypeDiscriminator": "FindKeysRange", + "LastKey": 0, + "KeyStep": 1, + "Limit": 0 + }, + "Flags": "RO, Access" + } + ] + }, + { + "Command": "ZPTTL", + "Name": "ZPTTL", + "Arity": -5, + "Flags": "Fast, ReadOnly", + "FirstKey": 1, + "LastKey": 1, + "Step": 1, + "AclCategories": "SortedSet, Fast, Read, Garnet", + "KeySpecifications": [ + { + "BeginSearch": { + "TypeDiscriminator": "BeginSearchIndex", + "Index": 1 + }, + "FindKeys": { + "TypeDiscriminator": "FindKeysRange", + "LastKey": 0, + "KeyStep": 1, + "Limit": 0 + }, + "Flags": "RO, Access" + } + ] + }, + { + "Command": "ZTTL", + "Name": "ZTTL", + "Arity": -5, + "Flags": "Fast, ReadOnly", + "FirstKey": 1, + "LastKey": 1, + "Step": 1, + "AclCategories": "SortedSet, Fast, Read, Garnet", + "KeySpecifications": [ + { + "BeginSearch": { + "TypeDiscriminator": "BeginSearchIndex", + "Index": 1 + }, + "FindKeys": { + "TypeDiscriminator": "FindKeysRange", + "LastKey": 0, + "KeyStep": 1, + "Limit": 0 + }, + "Flags": "RO, Access" + } + ] } ] \ No newline at end of file diff --git a/website/docs/commands/garnet-specific.md b/website/docs/commands/garnet-specific.md index 65a5bd4828..26cc9def4e 100644 --- a/website/docs/commands/garnet-specific.md +++ b/website/docs/commands/garnet-specific.md @@ -249,3 +249,238 @@ Below is the expected behavior of ETag-associated key-value pairs when non-ETag All other commands will update the etag internally if they modify the underlying data, and any responses from them will not expose the etag to the client. To the users the etag and it's updates remain hidden in non-etag commands. --- + +### ZEXPIRE + +#### Syntax + +```bash + ZEXPIRE key seconds [NX | XX | GT | LT] MEMBERS nummembers member [member ...] +``` + +Sets a timeout on one or more members of a sorted set key. After the timeout has expired, the members will automatically be deleted. The timeout is specified in seconds. + +The command supports several options to control when the expiration should be set: + +* **NX:** Only set expiry on members that have no existing expiry +* **XX:** Only set expiry on members that already have an expiry set +* **GT:** Only set expiry when it's greater than the current expiry +* **LT:** Only set expiry when it's less than the current expiry + +The **NX**, **XX**, **GT**, and **LT** options are mutually exclusive. + +#### Resp Reply + +Array reply: For each member, returns: + +* 1 if the timeout was set +* 0 if the member doesn't exist +* -1 if timeout was not set due to condition not being met + +--- + +### ZEXPIREAT + +#### Syntax + +```bash + ZEXPIREAT key unix-time-seconds [NX | XX | GT | LT] MEMBERS nummembers member [member ...] +``` + +Sets an absolute expiration time (Unix timestamp in seconds) for one or more sorted set members. After the timestamp has passed, the members will automatically be deleted. + +The command supports several options to control when the expiration should be set: + +* **NX:** Only set expiry on members that have no existing expiry +* **XX:** Only set expiry on members that already have an expiry set +* **GT:** Only set expiry when it's greater than the current expiry +* **LT:** Only set expiry when it's less than the current expiry + +The **NX**, **XX**, **GT**, and **LT** options are mutually exclusive. + +#### Resp Reply + +Array reply: For each member, returns: + +* 1 if the timeout was set +* 0 if the member doesn't exist +* -1 if timeout was not set due to condition not being met + +--- + +### ZPEXPIRE + +#### Syntax + +```bash + ZPEXPIRE key milliseconds [NX | XX | GT | LT] MEMBERS nummembers member [member ...] +``` + +Similar to HEXPIRE but the timeout is specified in milliseconds instead of seconds. + +The command supports several options to control when the expiration should be set: + +* **NX:** Only set expiry on members that have no existing expiry +* **XX:** Only set expiry on members that already have an expiry set +* **GT:** Only set expiry when it's greater than the current expiry +* **LT:** Only set expiry when it's less than the current expiry + +The **NX**, **XX**, **GT**, and **LT** options are mutually exclusive. + +#### Resp Reply + +Array reply: For each member, returns: + +* 1 if the timeout was set +* 0 if the member doesn't exist +* -1 if timeout was not set due to condition not being met + +--- + +### ZPEXPIREAT + +#### Syntax + +```bash + ZPEXPIREAT key unix-time-milliseconds [NX | XX | GT | LT] MEMBERS nummembers member [member ...] +``` + +Similar to HEXPIREAT but uses Unix timestamp in milliseconds instead of seconds. + +The command supports several options to control when the expiration should be set: + +* **NX:** Only set expiry on members that have no existing expiry +* **XX:** Only set expiry on members that already have an expiry set +* **GT:** Only set expiry when it's greater than the current expiry +* **LT:** Only set expiry when it's less than the current expiry + +The **NX**, **XX**, **GT**, and **LT** options are mutually exclusive. + +#### Resp Reply + +Array reply: For each member, returns: + +* 1 if the timeout was set +* 0 if the member doesn't exist +* -1 if timeout was not set due to condition not being met + +--- + +### ZTTL + +#### Syntax + +```bash + ZTTL key MEMBERS nummembers member [member ...] +``` + +Returns the remaining time to live in seconds for one or more sorted set members that have a timeout set. + +#### Resp Reply + +Array reply: For each member, returns: + +* TTL in seconds if the member exists and has an expiry set +* -1 if the member exists but has no expiry set +* -2 if the member does not exist + +--- + +### ZPTTL + +#### Syntax + +```bash + ZPTTL key MEMBERS nummembers member [member ...] +``` + +Similar to HTTL but returns the remaining time to live in milliseconds instead of seconds. + +#### Resp Reply + +Array reply: For each member, returns: + +* TTL in milliseconds if the member exists and has an expiry set +* -1 if the member exists but has no expiry set +* -2 if the member does not exist + +--- + +### ZEXPIRETIME + +#### Syntax + +```bash + ZEXPIRETIME key MEMBERS nummembers member [member ...] +``` + +Returns the absolute Unix timestamp (in seconds) at which the specified sorted set members will expire. + +#### Resp Reply + +Array reply: For each member, returns: + +* Unix timestamp in seconds when the member will expire +* -1 if the member exists but has no expiry set +* -2 if the member does not exist + +--- + +### ZPEXPIRETIME + +#### Syntax + +```bash + ZPEXPIRETIME key MEMBERS nummembers member [member ...] +``` + +Similar to HEXPIRETIME but returns the expiry timestamp in milliseconds instead of seconds. + +#### Resp Reply + +Array reply: For each member, returns: + +* Unix timestamp in milliseconds when the member will expire +* -1 if the member exists but has no expiry set +* -2 if the member does not exist + +--- + +### ZPERSIST + +#### Syntax + +```bash + ZPERSIST key MEMBERS nummembers member [member ...] +``` + +Removes the expiration from the specified sorted set members, making them persistent. + +#### Resp Reply + +Array reply: For each member, returns: + +* 1 if the timeout was removed +* 0 if the member exists but has no timeout +* -1 if the member does not exist + +--- + +### ZCOLLECT + +#### Syntax + +```bash + ZCOLLECT key [key ...] +``` + +Manualy trigger cleanup of expired member from memory for a given Hash set key. + +Use `*` as the key to collect it from all sorted set keys. + +#### Resp Reply + +Simple reply: OK response +Error reply: ERR ZCOLLECT scan already in progress + +--- \ No newline at end of file diff --git a/website/docs/getting-started/configuration.md b/website/docs/getting-started/configuration.md index 18bed2ffdb..6a6cc4951c 100644 --- a/website/docs/getting-started/configuration.md +++ b/website/docs/getting-started/configuration.md @@ -119,8 +119,7 @@ For all available command line settings, run `GarnetServer.exe -h` or `GarnetSer | **WaitForCommit** | ```--aof-commit-wait``` | ```bool``` | | Wait for AOF to flush the commit before returning results to client. Warning: will greatly increase operation latency. | | **AofSizeLimit** | ```--aof-size-limit``` | ```string``` | Memory size | Maximum size of AOF (rounds down to power of 2) after which unsafe truncation will be applied. Left empty AOF will grow without bound unless a checkpoint is taken | | **CompactionFrequencySecs** | ```--compaction-freq``` | ```int``` | Integer in range:
[0, MaxValue] | Background hybrid log compaction frequency in seconds. 0 = disabled (compaction performed before checkpointing instead) | -| **HashCollectFrequencySecs** | ```--hcollect-freq``` | ```int``` | Integer in range:
[0, MaxValue] | Frequency in seconds for the background task to perform Hash collection. 0 = disabled. Hash collect is used to delete expired fields from hash without waiting for a write operation. Use the HCOLLECT API to collect on-demand. | -| **SortedSetCollectFrequencySecs** | ```--zcollect-freq``` | ```int``` | Integer in range:
[0, MaxValue] | Frequency in seconds for the background task to perform Sorted Set collection. 0 = disabled. Sorted Set collect is used to delete expired members from Sorted Set without waiting for a write operation. Use the ZCOLLECT API to collect on-demand. | +| **ExpiredObjectCollectionFrequencySecs** | ```--expired-object-collection-freq``` | ```int``` | Integer in range:
[0, MaxValue] | Frequency in seconds for the background task to perform object collection which removes expired members within object from memory. 0 = disabled. Use the HCOLLECT and ZCOLLECT API to collect on-demand. | | **CompactionType** | ```--compaction-type``` | ```LogCompactionType``` | None, Shift, Scan, Lookup | Hybrid log compaction type. Value options: None - No compaction, Shift - shift begin address without compaction (data loss), Scan - scan old pages and move live records to tail (no data loss), Lookup - lookup each record in compaction range, for record liveness checking using hash chain (no data loss) | | **CompactionForceDelete** | ```--compaction-force-delete``` | ```bool``` | | Forcefully delete the inactive segments immediately after the compaction strategy (type) is applied. If false, take a checkpoint to actually delete the older data files from disk. | | **CompactionMaxSegments** | ```--compaction-max-segments``` | ```int``` | Integer in range:
[0, MaxValue] | Number of log segments created on disk before compaction triggers. | From 3ccd5055ff8d9aceff76e59c27090cb2e2bd0924 Mon Sep 17 00:00:00 2001 From: Vijay-Nirmal Date: Thu, 20 Feb 2025 01:42:29 +0530 Subject: [PATCH 7/9] Fixed test case failure --- libs/server/Resp/Parser/RespCommand.cs | 4 ++ test/Garnet.test/Resp/ACL/RespCommandTests.cs | 40 +++++++++---------- 2 files changed, 24 insertions(+), 20 deletions(-) diff --git a/libs/server/Resp/Parser/RespCommand.cs b/libs/server/Resp/Parser/RespCommand.cs index a8e07b785f..a0d52d2ea2 100644 --- a/libs/server/Resp/Parser/RespCommand.cs +++ b/libs/server/Resp/Parser/RespCommand.cs @@ -1611,6 +1611,10 @@ private RespCommand FastParseArrayCommand(ref int count, ref ReadOnlySpan { return RespCommand.PEXPIRETIME; } + else if (*(ulong*)(ptr + 2) == MemoryMarshal.Read("1\r\nHEXPI"u8) && *(ulong*)(ptr + 10) == MemoryMarshal.Read("RETIME\r\n"u8)) + { + return RespCommand.HEXPIRETIME; + } else if (*(ulong*)(ptr + 2) == MemoryMarshal.Read("1\r\nINCRB"u8) && *(ulong*)(ptr + 10) == MemoryMarshal.Read("YFLOAT\r\n"u8)) { return RespCommand.INCRBYFLOAT; diff --git a/test/Garnet.test/Resp/ACL/RespCommandTests.cs b/test/Garnet.test/Resp/ACL/RespCommandTests.cs index 8dc5f71271..e601da049a 100644 --- a/test/Garnet.test/Resp/ACL/RespCommandTests.cs +++ b/test/Garnet.test/Resp/ACL/RespCommandTests.cs @@ -6849,10 +6849,10 @@ public async Task ZExpireACLsAsync() { await CheckCommandsAsync( "ZEXPIRE", - [DoHExpireAsync] + [DoZExpireAsync] ); - static async Task DoHExpireAsync(GarnetClient client) + static async Task DoZExpireAsync(GarnetClient client) { var val = await client.ExecuteForStringArrayResultAsync("ZEXPIRE", ["foo", "1", "MEMBERS", "1", "bar"]); ClassicAssert.AreEqual(1, val.Length); @@ -6865,10 +6865,10 @@ public async Task ZPExpireACLsAsync() { await CheckCommandsAsync( "ZPEXPIRE", - [DoHPExpireAsync] + [DoZPExpireAsync] ); - static async Task DoHPExpireAsync(GarnetClient client) + static async Task DoZPExpireAsync(GarnetClient client) { var val = await client.ExecuteForStringArrayResultAsync("ZPEXPIRE", ["foo", "1", "MEMBERS", "1", "bar"]); ClassicAssert.AreEqual(1, val.Length); @@ -6881,10 +6881,10 @@ public async Task ZExpireAtACLsAsync() { await CheckCommandsAsync( "ZEXPIREAT", - [DoHExpireAtAsync] + [DoZExpireAtAsync] ); - static async Task DoHExpireAtAsync(GarnetClient client) + static async Task DoZExpireAtAsync(GarnetClient client) { var val = await client.ExecuteForStringArrayResultAsync("ZEXPIREAT", ["foo", DateTimeOffset.UtcNow.AddSeconds(3).ToUnixTimeSeconds().ToString(), "MEMBERS", "1", "bar"]); ClassicAssert.AreEqual(1, val.Length); @@ -6897,10 +6897,10 @@ public async Task ZPExpireAtACLsAsync() { await CheckCommandsAsync( "ZPEXPIREAT", - [DoHPExpireAtAsync] + [DoZPExpireAtAsync] ); - static async Task DoHPExpireAtAsync(GarnetClient client) + static async Task DoZPExpireAtAsync(GarnetClient client) { var val = await client.ExecuteForStringArrayResultAsync("ZPEXPIREAT", ["foo", DateTimeOffset.UtcNow.AddSeconds(3).ToUnixTimeMilliseconds().ToString(), "MEMBERS", "1", "bar"]); ClassicAssert.AreEqual(1, val.Length); @@ -6913,10 +6913,10 @@ public async Task ZExpireTimeACLsAsync() { await CheckCommandsAsync( "ZEXPIRETIME", - [DoHExpireTimeAsync] + [DoZExpireTimeAsync] ); - static async Task DoHExpireTimeAsync(GarnetClient client) + static async Task DoZExpireTimeAsync(GarnetClient client) { var val = await client.ExecuteForStringArrayResultAsync("ZEXPIRETIME", ["foo", "MEMBERS", "1", "bar"]); ClassicAssert.AreEqual(1, val.Length); @@ -6929,10 +6929,10 @@ public async Task ZPExpireTimeACLsAsync() { await CheckCommandsAsync( "ZPEXPIRETIME", - [DoHPExpireTimeAsync] + [DoZPExpireTimeAsync] ); - static async Task DoHPExpireTimeAsync(GarnetClient client) + static async Task DoZPExpireTimeAsync(GarnetClient client) { var val = await client.ExecuteForStringArrayResultAsync("ZPEXPIRETIME", ["foo", "MEMBERS", "1", "bar"]); ClassicAssert.AreEqual(1, val.Length); @@ -6945,10 +6945,10 @@ public async Task ZTTLACLsAsync() { await CheckCommandsAsync( "ZTTL", - [DoHETTLAsync] + [DoZETTLAsync] ); - static async Task DoHETTLAsync(GarnetClient client) + static async Task DoZETTLAsync(GarnetClient client) { var val = await client.ExecuteForStringArrayResultAsync("ZTTL", ["foo", "MEMBERS", "1", "bar"]); ClassicAssert.AreEqual(1, val.Length); @@ -6961,10 +6961,10 @@ public async Task ZPTTLACLsAsync() { await CheckCommandsAsync( "ZPTTL", - [DoHPETTLAsync] + [DoZPETTLAsync] ); - static async Task DoHPETTLAsync(GarnetClient client) + static async Task DoZPETTLAsync(GarnetClient client) { var val = await client.ExecuteForStringArrayResultAsync("ZPTTL", ["foo", "MEMBERS", "1", "bar"]); ClassicAssert.AreEqual(1, val.Length); @@ -6977,10 +6977,10 @@ public async Task ZPersistACLsAsync() { await CheckCommandsAsync( "ZPERSIST", - [DoHPersistAsync] + [DoZPersistAsync] ); - static async Task DoHPersistAsync(GarnetClient client) + static async Task DoZPersistAsync(GarnetClient client) { var val = await client.ExecuteForStringArrayResultAsync("ZPERSIST", ["foo", "MEMBERS", "1", "bar"]); ClassicAssert.AreEqual(1, val.Length); @@ -6993,10 +6993,10 @@ public async Task ZCollectACLsAsync() { await CheckCommandsAsync( "ZCOLLECT", - [DoHCollectAsync] + [DoZCollectAsync] ); - static async Task DoHCollectAsync(GarnetClient client) + static async Task DoZCollectAsync(GarnetClient client) { var val = await client.ExecuteForStringResultAsync("ZCOLLECT", ["foo"]); ClassicAssert.AreEqual("OK", val); From 9e1a5a9ddfdc5f41fcc887947d541f49cdf4c763 Mon Sep 17 00:00:00 2001 From: Vijay-Nirmal Date: Thu, 20 Feb 2025 12:15:42 +0530 Subject: [PATCH 8/9] Fixed slot test case --- .../RedirectTests/ClusterSlotVerificationTests.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/test/Garnet.test.cluster/RedirectTests/ClusterSlotVerificationTests.cs b/test/Garnet.test.cluster/RedirectTests/ClusterSlotVerificationTests.cs index 529ae523bd..2fa5925b75 100644 --- a/test/Garnet.test.cluster/RedirectTests/ClusterSlotVerificationTests.cs +++ b/test/Garnet.test.cluster/RedirectTests/ClusterSlotVerificationTests.cs @@ -119,6 +119,7 @@ public class ClusterSlotVerificationTests new ZUNION(), new ZUNIONSTORE(), new HEXPIRE(), + new ZEXPIRE(), new ZPEXPIRE(), new ZEXPIREAT(), new ZPEXPIREAT(), From a9c8b5a30f2448d2c81defce04d4abb6932724a1 Mon Sep 17 00:00:00 2001 From: Vijay-Nirmal Date: Mon, 3 Mar 2025 11:50:21 +0530 Subject: [PATCH 9/9] Fixed review commands --- libs/server/API/GarnetApiObjectCommands.cs | 20 + libs/server/API/GarnetWatchApi.cs | 7 + libs/server/API/IGarnetApi.cs | 42 ++ .../ItemBroker/CollectionItemBroker.cs | 8 +- .../Objects/SortedSet/SortedSetObject.cs | 107 ++-- .../Objects/SortedSet/SortedSetObjectImpl.cs | 5 +- libs/server/Resp/Objects/ObjectStoreUtils.cs | 47 ++ libs/server/Resp/Objects/SortedSetCommands.cs | 55 +- .../Storage/Session/ObjectStore/Common.cs | 15 + .../Session/ObjectStore/SortedSetOps.cs | 144 ++++++ libs/server/StoreWrapper.cs | 6 +- .../GarnetCommandsDocs.json | 470 ++++++++++++++++++ test/Garnet.test/TestProcedureSortedSets.cs | 16 + 13 files changed, 878 insertions(+), 64 deletions(-) diff --git a/libs/server/API/GarnetApiObjectCommands.cs b/libs/server/API/GarnetApiObjectCommands.cs index c079fe5858..8213a3fb1e 100644 --- a/libs/server/API/GarnetApiObjectCommands.cs +++ b/libs/server/API/GarnetApiObjectCommands.cs @@ -173,18 +173,38 @@ public GarnetStatus SortedSetIntersectStore(ArgSlice destinationKey, ReadOnlySpa public GarnetStatus SortedSetExpire(ArgSlice key, long expireAt, bool isMilliseconds, ExpireOption expireOption, ref ObjectInput input, ref GarnetObjectStoreOutput outputFooter) => storageSession.SortedSetExpire(key, expireAt, isMilliseconds, expireOption, ref input, ref outputFooter, ref objectContext); + /// + public GarnetStatus SortedSetExpire(ArgSlice key, ReadOnlySpan members, DateTimeOffset expireAt, ExpireOption expireOption, out int[] results) + => storageSession.SortedSetExpire(key, members, expireAt, expireOption, out results, ref objectContext); + /// public GarnetStatus SortedSetPersist(ArgSlice key, ref ObjectInput input, ref GarnetObjectStoreOutput outputFooter) => storageSession.SortedSetPersist(key, ref input, ref outputFooter, ref objectContext); + /// + public GarnetStatus SortedSetPersist(ArgSlice key, ReadOnlySpan members, out int[] results) + => storageSession.SortedSetPersist(key, members, out results, ref objectContext); + /// public GarnetStatus SortedSetTimeToLive(ArgSlice key, bool isMilliseconds, bool isTimestamp, ref ObjectInput input, ref GarnetObjectStoreOutput outputFooter) => storageSession.SortedSetTimeToLive(key, isMilliseconds, isTimestamp, ref input, ref outputFooter, ref objectContext); + /// + public GarnetStatus SortedSetTimeToLive(ArgSlice key, ReadOnlySpan members, out TimeSpan[] expireIn) + => storageSession.SortedSetTimeToLive(key, members, out expireIn, ref objectContext); + /// public GarnetStatus SortedSetCollect(ReadOnlySpan keys, ref ObjectInput input) => storageSession.SortedSetCollect(keys, ref input, ref objectContext); + /// + public GarnetStatus SortedSetCollect() + => storageSession.SortedSetCollect(ref objectContext); + + /// + public GarnetStatus SortedSetCollect(ReadOnlySpan keys) + => storageSession.SortedSetCollect(keys, ref objectContext); + #endregion #region Geospatial commands diff --git a/libs/server/API/GarnetWatchApi.cs b/libs/server/API/GarnetWatchApi.cs index 3dc0e2db68..a511623615 100644 --- a/libs/server/API/GarnetWatchApi.cs +++ b/libs/server/API/GarnetWatchApi.cs @@ -250,6 +250,13 @@ public GarnetStatus SortedSetTimeToLive(ArgSlice key, bool isMilliseconds, bool return garnetApi.SortedSetTimeToLive(key, isMilliseconds, isTimestamp, ref input, ref outputFooter); } + /// + public GarnetStatus SortedSetTimeToLive(ArgSlice key, ReadOnlySpan members, out TimeSpan[] expireIn) + { + garnetApi.WATCH(key, StoreType.Object); + return garnetApi.SortedSetTimeToLive(key, members, out expireIn); + } + #endregion #region List Methods diff --git a/libs/server/API/IGarnetApi.cs b/libs/server/API/IGarnetApi.cs index 4f913f2098..36734a4d92 100644 --- a/libs/server/API/IGarnetApi.cs +++ b/libs/server/API/IGarnetApi.cs @@ -563,6 +563,17 @@ public interface IGarnetApi : IGarnetReadApi, IGarnetAdvancedApi /// The status of the operation. GarnetStatus SortedSetExpire(ArgSlice key, long expireAt, bool isMilliseconds, ExpireOption expireOption, ref ObjectInput input, ref GarnetObjectStoreOutput outputFooter); + /// + /// Sets an expiration time on a sorted set member. + /// + /// The key of the sorted set. + /// The members to set expiration for. + /// The expiration time. + /// The expiration option to apply. + /// The results of the operation. + /// The status of the operation. + GarnetStatus SortedSetExpire(ArgSlice key, ReadOnlySpan members, DateTimeOffset expireAt, ExpireOption expireOption, out int[] results); + /// /// Persists the specified sorted set member, removing any expiration time set on it. /// @@ -572,6 +583,15 @@ public interface IGarnetApi : IGarnetReadApi, IGarnetAdvancedApi /// The status of the operation. GarnetStatus SortedSetPersist(ArgSlice key, ref ObjectInput input, ref GarnetObjectStoreOutput outputFooter); + /// + /// Persists the specified sorted set members, removing any expiration time set on them. + /// + /// The key of the sorted set. + /// The members to persist. + /// The results of the operation. + /// The status of the operation. + GarnetStatus SortedSetPersist(ArgSlice key, ReadOnlySpan members, out int[] results); + /// /// Deletes already expired members from the sorted set. /// @@ -580,6 +600,19 @@ public interface IGarnetApi : IGarnetReadApi, IGarnetAdvancedApi /// The status of the operation. GarnetStatus SortedSetCollect(ReadOnlySpan keys, ref ObjectInput input); + /// + /// Collects expired elements from the sorted set. + /// + /// The status of the operation. + GarnetStatus SortedSetCollect(); + + /// + /// Collects expired elements from the sorted set for the specified keys. + /// + /// The keys of the sorted sets to collect expired elements from. + /// The status of the operation. + GarnetStatus SortedSetCollect(ReadOnlySpan keys); + #endregion #region Set Methods @@ -1445,6 +1478,15 @@ public interface IGarnetReadApi /// The status of the operation. GarnetStatus SortedSetTimeToLive(ArgSlice key, bool isMilliseconds, bool isTimestamp, ref ObjectInput input, ref GarnetObjectStoreOutput outputFooter); + /// + /// Returns the time to live for a sorted set members. + /// + /// The key of the sorted set. + /// The members to get the time to live for. + /// The output array containing the time to live for each member. + /// The status of the operation. + GarnetStatus SortedSetTimeToLive(ArgSlice key, ReadOnlySpan members, out TimeSpan[] expireIn); + #endregion #region Geospatial Methods diff --git a/libs/server/Objects/ItemBroker/CollectionItemBroker.cs b/libs/server/Objects/ItemBroker/CollectionItemBroker.cs index 289ea1b0ff..f33fc61a4b 100644 --- a/libs/server/Objects/ItemBroker/CollectionItemBroker.cs +++ b/libs/server/Objects/ItemBroker/CollectionItemBroker.cs @@ -399,11 +399,11 @@ private static bool TryMoveNextListItem(ListObject srcListObj, ListObject dstLis /// BZPOPMIN and BZPOPMAX share same implementation since Dictionary.First() and Last() /// handle the ordering automatically based on sorted set scores /// - private static unsafe bool TryGetNextSetObjects(byte[] key, SortedSetObject sortedSetObj, RespCommand command, ArgSlice[] cmdArgs, out CollectionItemResult result) + private static unsafe bool TryGetNextSetObjects(byte[] key, SortedSetObject sortedSetObj, int count, RespCommand command, ArgSlice[] cmdArgs, out CollectionItemResult result) { result = default; - if (sortedSetObj.Count() == 0) return false; + if (count == 0) return false; switch (command) { @@ -416,7 +416,7 @@ private static unsafe bool TryGetNextSetObjects(byte[] key, SortedSetObject sort case RespCommand.BZMPOP: var lowScoresFirst = *(bool*)cmdArgs[0].ptr; var popCount = *(int*)cmdArgs[1].ptr; - popCount = Math.Min(popCount, sortedSetObj.Count()); + popCount = Math.Min(popCount, count); var scores = new double[popCount]; var items = new byte[popCount][]; @@ -552,7 +552,7 @@ private unsafe bool TryGetResult(byte[] key, StorageSession storageSession, Resp if (currCount == 0) return false; - return TryGetNextSetObjects(key, setObj, command, cmdArgs, out result); + return TryGetNextSetObjects(key, setObj, currCount, command, cmdArgs, out result); default: return false; diff --git a/libs/server/Objects/SortedSet/SortedSetObject.cs b/libs/server/Objects/SortedSet/SortedSetObject.cs index 3254d1117b..e2a583e822 100644 --- a/libs/server/Objects/SortedSet/SortedSetObject.cs +++ b/libs/server/Objects/SortedSet/SortedSetObject.cs @@ -138,7 +138,7 @@ public enum SortedSetOrderOperation /// /// Sorted Set /// - public unsafe partial class SortedSetObject : GarnetObjectBase + public partial class SortedSetObject : GarnetObjectBase { private readonly SortedSet<(double Score, byte[] Element)> sortedSet; private readonly Dictionary sortedSetDict; @@ -175,41 +175,42 @@ public SortedSetObject(BinaryReader reader) keyLength &= ~ExpirationBitMask; var item = reader.ReadBytes(keyLength); var score = reader.ReadDouble(); + var canAddItem = true; + long expiration = 0; if (hasExpiration) { - var expiration = reader.ReadInt64(); - var isExpired = expiration < DateTimeOffset.UtcNow.Ticks; - if (!isExpired) + expiration = reader.ReadInt64(); + canAddItem = expiration >= DateTimeOffset.UtcNow.Ticks; + } + + if (canAddItem) + { + sortedSetDict.Add(item, score); + sortedSet.Add((score, item)); + this.UpdateSize(item); + + if (expiration > 0) { - sortedSetDict.Add(item, score); - sortedSet.Add((score, item)); InitializeExpirationStructures(); expirationTimes.Add(item, expiration); expirationQueue.Enqueue(item, expiration); UpdateExpirationSize(item, true); } } - else - { - sortedSetDict.Add(item, score); - sortedSet.Add((score, item)); - } - - this.UpdateSize(item); } } /// /// Copy constructor /// - public SortedSetObject(SortedSet<(double, byte[])> sortedSet, Dictionary sortedSetDict, Dictionary expirationTimes, PriorityQueue expirationQueue, long expiration, long size) - : base(expiration, size) + public SortedSetObject(SortedSetObject sortedSetObject) + : base(sortedSetObject.Expiration, sortedSetObject.Size) { - this.sortedSet = sortedSet; - this.sortedSetDict = sortedSetDict; - this.expirationTimes = expirationTimes; - this.expirationQueue = expirationQueue; + this.sortedSet = sortedSetObject.sortedSet; + this.sortedSetDict = sortedSetObject.sortedSetDict; + this.expirationTimes = sortedSetObject.expirationTimes; + this.expirationQueue = sortedSetObject.expirationQueue; } /// @@ -290,12 +291,23 @@ public void Add(byte[] item, double score) /// public bool Equals(SortedSetObject other) { - // TODO: Implement equals with expiration times - if (sortedSetDict.Count != other.sortedSetDict.Count) return false; + if (sortedSetDict.Count() != other.sortedSetDict.Count()) return false; foreach (var key in sortedSetDict) + { + if (IsExpired(key.Key) && IsExpired(key.Key)) + { + continue; + } + + if (IsExpired(key.Key) || IsExpired(key.Key)) + { + return false; + } + if (!other.sortedSetDict.TryGetValue(key.Key, out var otherValue) || key.Value != otherValue) return false; + } return true; } @@ -304,7 +316,7 @@ public bool Equals(SortedSetObject other) public override void Dispose() { } /// - public override GarnetObjectBase Clone() => new SortedSetObject(sortedSet, sortedSetDict, expirationTimes, expirationQueue, Expiration, Size); + public override GarnetObjectBase Clone() => new SortedSetObject(this); /// public override unsafe bool Operate(ref ObjectInput input, ref GarnetObjectStoreOutput output, out long sizeChange) @@ -559,6 +571,12 @@ public static void InPlaceDiff(Dictionary dict1, SortedSetObject } } + /// + /// Tries to get the score of the specified key. + /// + /// The key to get the score for. + /// The score of the key if found. + /// True if the key is found and not expired; otherwise, false. [MethodImpl(MethodImplOptions.AggressiveInlining)] public bool TryGetScore(byte[] key, out double value) { @@ -571,9 +589,13 @@ public bool TryGetScore(byte[] key, out double value) return sortedSetDict.TryGetValue(key, out value); } + /// + /// Gets the count of elements in the sorted set. + /// + /// The count of elements in the sorted set. public int Count() { - if (expirationTimes is null) + if (!HasExpirableItems()) { return sortedSetDict.Count; } @@ -589,9 +611,18 @@ public int Count() return sortedSetDict.Count - expiredKeysCount; } + /// + /// Determines whether the specified key is expired. + /// + /// The key to check for expiration. + /// True if the key is expired; otherwise, false. [MethodImpl(MethodImplOptions.AggressiveInlining)] public bool IsExpired(byte[] key) => expirationTimes is not null && expirationTimes.TryGetValue(key, out var expiration) && expiration < DateTimeOffset.UtcNow.Ticks; + /// + /// Determines whether the sorted set has expirable items. + /// + /// True if the sorted set has expirable items; otherwise, false. [MethodImpl(MethodImplOptions.AggressiveInlining)] public bool HasExpirableItems() { @@ -662,7 +693,7 @@ private int SetExpiration(byte[] key, long expiration, ExpireOption expireOption { if (!sortedSetDict.ContainsKey(key)) { - return (int)ExpireResult.KeyNotFound; + return (int)SortedSetExpireResult.KeyNotFound; } if (expiration <= DateTimeOffset.UtcNow.Ticks) @@ -670,7 +701,7 @@ private int SetExpiration(byte[] key, long expiration, ExpireOption expireOption sortedSetDict.Remove(key, out var value); sortedSet.Remove((value, key)); UpdateSize(key, false); - return (int)ExpireResult.KeyAlreadyExpired; + return (int)SortedSetExpireResult.KeyAlreadyExpired; } InitializeExpirationStructures(); @@ -681,7 +712,7 @@ private int SetExpiration(byte[] key, long expiration, ExpireOption expireOption (expireOption.HasFlag(ExpireOption.GT) && expiration <= currentExpiration) || (expireOption.HasFlag(ExpireOption.LT) && expiration >= currentExpiration)) { - return (int)ExpireResult.ExpireConditionNotMet; + return (int)SortedSetExpireResult.ExpireConditionNotMet; } expirationTimes[key] = expiration; @@ -692,7 +723,7 @@ private int SetExpiration(byte[] key, long expiration, ExpireOption expireOption { if (expireOption.HasFlag(ExpireOption.XX) || expireOption.HasFlag(ExpireOption.GT)) { - return (int)ExpireResult.ExpireConditionNotMet; + return (int)SortedSetExpireResult.ExpireConditionNotMet; } expirationTimes[key] = expiration; @@ -700,7 +731,7 @@ private int SetExpiration(byte[] key, long expiration, ExpireOption expireOption UpdateExpirationSize(key); } - return (int)ExpireResult.ExpireUpdated; + return (int)SortedSetExpireResult.ExpireUpdated; } private int Persist(byte[] key) @@ -775,11 +806,29 @@ private void UpdateSize(ReadOnlySpan item, bool add = true) Debug.Assert(this.Size >= MemoryUtils.SortedSetOverhead + MemoryUtils.DictionaryOverhead); } - enum ExpireResult + /// + /// Result of an expiration operation. + /// + enum SortedSetExpireResult { + /// + /// The key was not found. + /// KeyNotFound = -2, + + /// + /// The expiration condition was not met. + /// ExpireConditionNotMet = 0, + + /// + /// The expiration was updated. + /// ExpireUpdated = 1, + + /// + /// The key was already expired. + /// KeyAlreadyExpired = 2, } } diff --git a/libs/server/Objects/SortedSet/SortedSetObjectImpl.cs b/libs/server/Objects/SortedSet/SortedSetObjectImpl.cs index b819aaf448..8176814111 100644 --- a/libs/server/Objects/SortedSet/SortedSetObjectImpl.cs +++ b/libs/server/Objects/SortedSet/SortedSetObjectImpl.cs @@ -370,6 +370,8 @@ private void SortedSetCount(ref ObjectInput input, ref SpanByteAndMemory output) private void SortedSetIncrement(ref ObjectInput input, ref SpanByteAndMemory output) { + DeleteExpiredItems(); + // ZINCRBY key increment member var isMemory = false; MemoryHandle ptrHandle = default; @@ -395,8 +397,7 @@ private void SortedSetIncrement(ref ObjectInput input, ref SpanByteAndMemory out if (sortedSetDict.TryGetValue(member, out var score)) { - score = IsExpired(member) ? 0 : score; - sortedSetDict[member] = score + incrValue; + sortedSetDict[member] += incrValue; sortedSet.Remove((score, member)); sortedSet.Add((sortedSetDict[member], member)); } diff --git a/libs/server/Resp/Objects/ObjectStoreUtils.cs b/libs/server/Resp/Objects/ObjectStoreUtils.cs index 407ebf5ebc..4264b1c189 100644 --- a/libs/server/Resp/Objects/ObjectStoreUtils.cs +++ b/libs/server/Resp/Objects/ObjectStoreUtils.cs @@ -39,6 +39,53 @@ private bool AbortWithErrorMessage(ReadOnlySpan errorMessage) return true; } + /// + /// Aborts the execution of the current object store command and outputs a given error message. + /// + /// The format string for the error message. + /// The first argument to format. + /// true if the command was completely consumed, false if the input on the receive buffer was incomplete. + private bool AbortWithErrorMessage(string format, object arg0) + { + return AbortWithErrorMessage(Encoding.ASCII.GetBytes(string.Format(format, arg0))); + } + + /// + /// Aborts the execution of the current object store command and outputs a given error message. + /// + /// The format string for the error message. + /// The first argument to format. + /// The second argument to format. + /// true if the command was completely consumed, false if the input on the receive buffer was incomplete. + private bool AbortWithErrorMessage(string format, object arg0, object arg1) + { + return AbortWithErrorMessage(Encoding.ASCII.GetBytes(string.Format(format, arg0, arg1))); + } + + /// + /// Aborts the execution of the current object store command and outputs a given error message. + /// + /// The format string for the error message. + /// The first argument to format. + /// The second argument to format. + /// The third argument to format. + /// true if the command was completely consumed, false if the input on the receive buffer was incomplete. + private bool AbortWithErrorMessage(string format, object arg0, object arg1, object arg2) + { + return AbortWithErrorMessage(Encoding.ASCII.GetBytes(string.Format(format, arg0, arg1, arg2))); + } + + /// + /// Aborts the execution of the current object store command and outputs a given error message. + /// + /// The format string for the error message. + /// The arguments to format. + /// true if the command was completely consumed, false if the input on the receive buffer was incomplete. + private bool AbortWithErrorMessage(string format, params object[] args) + { + return AbortWithErrorMessage(Encoding.ASCII.GetBytes(string.Format(format, args))); + } + /// /// Tries to parse the input as "LEFT" or "RIGHT" and returns the corresponding OperationDirection. /// If parsing fails, returns OperationDirection.Unknown. diff --git a/libs/server/Resp/Objects/SortedSetCommands.cs b/libs/server/Resp/Objects/SortedSetCommands.cs index 9276a04490..98ab45e5f1 100644 --- a/libs/server/Resp/Objects/SortedSetCommands.cs +++ b/libs/server/Resp/Objects/SortedSetCommands.cs @@ -425,7 +425,7 @@ private unsafe bool SortedSetMPop(ref TGarnetApi storageApi) if (numKeys < 0) { - return AbortWithErrorMessage(Encoding.ASCII.GetBytes(string.Format(CmdStrings.GenericParamShouldBeGreaterThanZero, "numkeys"))); + return AbortWithErrorMessage(CmdStrings.GenericParamShouldBeGreaterThanZero, "numkeys"); } // Validate we have enough arguments (no of keys + (MIN or MAX)) @@ -470,7 +470,7 @@ private unsafe bool SortedSetMPop(ref TGarnetApi storageApi) if (count < 0) { - return AbortWithErrorMessage(Encoding.ASCII.GetBytes(string.Format(CmdStrings.GenericParamShouldBeGreaterThanZero, "count"))); + return AbortWithErrorMessage(CmdStrings.GenericParamShouldBeGreaterThanZero, "count"); } } @@ -1066,7 +1066,7 @@ private unsafe bool SortedSetIntersect(ref TGarnetApi storageApi) if (nKeys < 1) { - return AbortWithErrorMessage(Encoding.ASCII.GetBytes(string.Format(CmdStrings.GenericErrAtLeastOneKey, nameof(RespCommand.ZINTER)))); + return AbortWithErrorMessage(CmdStrings.GenericErrAtLeastOneKey, nameof(RespCommand.ZINTER)); } if (parseState.Count < nKeys + 1) @@ -1105,7 +1105,7 @@ private unsafe bool SortedSetIntersect(ref TGarnetApi storageApi) { if (!parseState.TryGetDouble(currentArg + i, out weights[i])) { - return AbortWithErrorMessage(Encoding.ASCII.GetBytes(string.Format(CmdStrings.GenericErrNotAFloat, "weight"))); + return AbortWithErrorMessage(CmdStrings.GenericErrNotAFloat, "weight"); } } currentArg += nKeys; @@ -1181,7 +1181,7 @@ private unsafe bool SortedSetIntersectLength(ref TGarnetApi storageA if (nKeys < 1) { - return AbortWithErrorMessage(Encoding.ASCII.GetBytes(string.Format(CmdStrings.GenericErrAtLeastOneKey, nameof(RespCommand.ZINTERCARD)))); + return AbortWithErrorMessage(CmdStrings.GenericErrAtLeastOneKey, nameof(RespCommand.ZINTERCARD)); } if (parseState.Count < nKeys + 1) @@ -1208,7 +1208,7 @@ private unsafe bool SortedSetIntersectLength(ref TGarnetApi storageA if (limitVal < 0) { - return AbortWithErrorMessage(Encoding.ASCII.GetBytes(string.Format(CmdStrings.GenericErrCantBeNegative, "LIMIT"))); + return AbortWithErrorMessage(CmdStrings.GenericErrCantBeNegative, "LIMIT"); } limit = limitVal; @@ -1277,7 +1277,7 @@ private unsafe bool SortedSetIntersectStore(ref TGarnetApi storageAp { if (!parseState.TryGetDouble(currentArg + i, out weights[i])) { - return AbortWithErrorMessage(Encoding.ASCII.GetBytes(string.Format(CmdStrings.GenericErrNotAFloat, "weight"))); + return AbortWithErrorMessage(CmdStrings.GenericErrNotAFloat, "weight"); } } currentArg += nKeys; @@ -1341,7 +1341,7 @@ private unsafe bool SortedSetUnion(ref TGarnetApi storageApi) if (nKeys < 1) { - return AbortWithErrorMessage(Encoding.ASCII.GetBytes(string.Format(CmdStrings.GenericErrAtLeastOneKey, nameof(RespCommand.ZUNION)))); + return AbortWithErrorMessage(CmdStrings.GenericErrAtLeastOneKey, nameof(RespCommand.ZUNION)); } if (parseState.Count < nKeys + 1) @@ -1376,7 +1376,7 @@ private unsafe bool SortedSetUnion(ref TGarnetApi storageApi) { if (!parseState.TryGetDouble(currentArg + i, out weights[i])) { - return AbortWithErrorMessage(Encoding.ASCII.GetBytes(string.Format(CmdStrings.GenericErrNotAFloat, "weight"))); + return AbortWithErrorMessage(CmdStrings.GenericErrNotAFloat, "weight"); } } currentArg += nKeys; @@ -1465,7 +1465,7 @@ private unsafe bool SortedSetUnionStore(ref TGarnetApi storageApi) if (nKeys < 1) { - return AbortWithErrorMessage(Encoding.ASCII.GetBytes(string.Format(CmdStrings.GenericErrAtLeastOneKey, nameof(RespCommand.ZUNIONSTORE)))); + return AbortWithErrorMessage(CmdStrings.GenericErrAtLeastOneKey, nameof(RespCommand.ZUNIONSTORE)); } if (parseState.Count < nKeys + 2) @@ -1494,7 +1494,7 @@ private unsafe bool SortedSetUnionStore(ref TGarnetApi storageApi) { if (!parseState.TryGetDouble(currentArg + i, out weights[i])) { - return AbortWithErrorMessage(Encoding.ASCII.GetBytes(string.Format(CmdStrings.GenericErrNotAFloat, "weight"))); + return AbortWithErrorMessage(CmdStrings.GenericErrNotAFloat, "weight"); } } currentArg += nKeys; @@ -1613,8 +1613,7 @@ private unsafe bool SortedSetBlockingMPop() // Read count of keys if (!parseState.TryGetInt(currTokenId++, out var numKeys)) { - var err = string.Format(CmdStrings.GenericParamShouldBeGreaterThanZero, "numkeys"); - return AbortWithErrorMessage(Encoding.ASCII.GetBytes(err)); + return AbortWithErrorMessage(CmdStrings.GenericParamShouldBeGreaterThanZero, "numkeys"); } // Should have MAX|MIN or it should contain COUNT + value @@ -1659,8 +1658,7 @@ private unsafe bool SortedSetBlockingMPop() if (!parseState.TryGetInt(currTokenId, out popCount) || popCount < 1) { - var err = string.Format(CmdStrings.GenericParamShouldBeGreaterThanZero, "count"); - return AbortWithErrorMessage(Encoding.ASCII.GetBytes(err)); + return AbortWithErrorMessage(CmdStrings.GenericParamShouldBeGreaterThanZero, "count"); } } @@ -1700,6 +1698,10 @@ private unsafe bool SortedSetBlockingMPop() /// /// Sets an expiration time for a member in the SortedSet stored at key. + /// ZEXPIRE key seconds [NX | XX | GT | LT] MEMBERS nummembers member [member ...] + /// ZPEXPIRE key milliseconds [NX | XX | GT | LT] MEMBERS nummembers member [member ...] + /// ZEXPIREAT key unix-time-seconds [NX | XX | GT | LT] MEMBERS nummembers member [member ...] + /// ZPEXPIREAT key unix-time-milliseconds [NX | XX | GT | LT] MEMBERS nummembers member [member ...] /// /// /// @@ -1756,17 +1758,17 @@ private unsafe bool SortedSetExpire(RespCommand command, ref TGarnet var fieldOption = parseState.GetArgSliceByRef(currIdx++); if (!fieldOption.ReadOnlySpan.EqualsUpperCaseSpanIgnoringCase(CmdStrings.MEMBERS)) { - return AbortWithErrorMessage(Encoding.ASCII.GetBytes(string.Format(CmdStrings.GenericErrMandatoryMissing, "MEMBERS"))); + return AbortWithErrorMessage(CmdStrings.GenericErrMandatoryMissing, "MEMBERS"); } if (!parseState.TryGetInt(currIdx++, out var numMembers)) { - return AbortWithErrorMessage(Encoding.ASCII.GetBytes(string.Format(CmdStrings.GenericParamShouldBeGreaterThanZero, "numMembers"))); + return AbortWithErrorMessage(CmdStrings.GenericParamShouldBeGreaterThanZero, "numMembers"); } if (parseState.Count != currIdx + numMembers) { - return AbortWithErrorMessage(Encoding.ASCII.GetBytes(string.Format(CmdStrings.GenericErrMustMatchNoOfArgs, "numMembers"))); + return AbortWithErrorMessage(CmdStrings.GenericErrMustMatchNoOfArgs, "numMembers"); } var membersParseState = parseState.Slice(currIdx, numMembers); @@ -1803,6 +1805,10 @@ private unsafe bool SortedSetExpire(RespCommand command, ref TGarnet /// /// Returns the time to live (TTL) for the specified members in the SortedSet stored at the given key. + /// ZTTL key MEMBERS nummembers member [member ...] + /// ZPTTL key MEMBERS nummembers member [member ...] + /// ZEXPIRETIME key MEMBERS nummembers member [member ...] + /// ZPEXPIRETIME key MEMBERS nummembers member [member ...] /// /// The type of the storage API. /// The RESP command indicating the type of TTL operation. @@ -1825,17 +1831,17 @@ private unsafe bool SortedSetTimeToLive(RespCommand command, ref TGa var fieldOption = parseState.GetArgSliceByRef(1); if (!fieldOption.ReadOnlySpan.EqualsUpperCaseSpanIgnoringCase(CmdStrings.MEMBERS)) { - return AbortWithErrorMessage(Encoding.ASCII.GetBytes(string.Format(CmdStrings.GenericErrMandatoryMissing, "MEMBERS"))); + return AbortWithErrorMessage(CmdStrings.GenericErrMandatoryMissing, "MEMBERS"); } if (!parseState.TryGetInt(2, out var numMembers)) { - return AbortWithErrorMessage(Encoding.ASCII.GetBytes(string.Format(CmdStrings.GenericParamShouldBeGreaterThanZero, "numMembers"))); + return AbortWithErrorMessage(CmdStrings.GenericParamShouldBeGreaterThanZero, "numMembers"); } if (parseState.Count != 3 + numMembers) { - return AbortWithErrorMessage(Encoding.ASCII.GetBytes(string.Format(CmdStrings.GenericErrMustMatchNoOfArgs, "numMembers"))); + return AbortWithErrorMessage(CmdStrings.GenericErrMustMatchNoOfArgs, "numMembers"); } var isMilliseconds = false; @@ -1892,6 +1898,7 @@ private unsafe bool SortedSetTimeToLive(RespCommand command, ref TGa /// /// Removes the expiration time from the specified members in the sorted set stored at the given key. + /// ZPERSIST key MEMBERS nummembers member [member ...] /// /// The type of the storage API. /// The storage API instance to interact with the underlying storage. @@ -1913,17 +1920,17 @@ private unsafe bool SortedSetPersist(ref TGarnetApi storageApi) var fieldOption = parseState.GetArgSliceByRef(1); if (!fieldOption.ReadOnlySpan.EqualsUpperCaseSpanIgnoringCase(CmdStrings.MEMBERS)) { - return AbortWithErrorMessage(Encoding.ASCII.GetBytes(string.Format(CmdStrings.GenericErrMandatoryMissing, "MEMBERS"))); + return AbortWithErrorMessage(CmdStrings.GenericErrMandatoryMissing, "MEMBERS"); } if (!parseState.TryGetInt(2, out var numMembers)) { - return AbortWithErrorMessage(Encoding.ASCII.GetBytes(string.Format(CmdStrings.GenericParamShouldBeGreaterThanZero, "numMembers"))); + return AbortWithErrorMessage(CmdStrings.GenericParamShouldBeGreaterThanZero, "numMembers"); } if (parseState.Count != 3 + numMembers) { - return AbortWithErrorMessage(Encoding.ASCII.GetBytes(string.Format(CmdStrings.GenericErrMustMatchNoOfArgs, "numMembers"))); + return AbortWithErrorMessage(CmdStrings.GenericErrMustMatchNoOfArgs, "numMembers"); } var membersParseState = parseState.Slice(3, numMembers); diff --git a/libs/server/Storage/Session/ObjectStore/Common.cs b/libs/server/Storage/Session/ObjectStore/Common.cs index ff993dabf3..b4284adaad 100644 --- a/libs/server/Storage/Session/ObjectStore/Common.cs +++ b/libs/server/Storage/Session/ObjectStore/Common.cs @@ -4,8 +4,10 @@ using System; using System.Diagnostics; using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; using System.Text; using Garnet.common; +using Garnet.common.Parsing; using Tsavorite.core; namespace Garnet.server @@ -334,11 +336,24 @@ unsafe int[] ProcessRespIntegerArrayOutput(GarnetObjectStoreOutput outputFooter, elements = new int[arraySize]; for (int i = 0; i < elements.Length; i++) { + if (*refPtr != ':') + { + RespParsingException.ThrowUnexpectedToken(*refPtr); + } + refPtr++; + element = null; if (RespReadUtils.TryReadInt32(ref refPtr, outputPtr + outputSpan.Length, out var number, out var _)) { elements[i] = number; } + + if (*(ushort*)refPtr != MemoryMarshal.Read("\r\n"u8)) + { + RespParsingException.ThrowUnexpectedToken(*refPtr); + } + + refPtr += 2; } } } diff --git a/libs/server/Storage/Session/ObjectStore/SortedSetOps.cs b/libs/server/Storage/Session/ObjectStore/SortedSetOps.cs index 6957e1f8cb..deb58fbfcb 100644 --- a/libs/server/Storage/Session/ObjectStore/SortedSetOps.cs +++ b/libs/server/Storage/Session/ObjectStore/SortedSetOps.cs @@ -5,12 +5,16 @@ using System.Collections.Generic; using System.Diagnostics; using System.Globalization; +using System.Linq; using System.Text; +using System.Xml.Linq; using Garnet.common; +using HdrHistogram; using Tsavorite.core; namespace Garnet.server { + using static System.Runtime.InteropServices.JavaScript.JSType; using ObjectStoreAllocator = GenericAllocator>>; using ObjectStoreFunctions = StoreFunctions>; @@ -1510,6 +1514,45 @@ public GarnetStatus SortedSetExpire(ArgSlice key, long expireAt, return RMWObjectStoreOperationWithOutput(key.ToArray(), ref innerInput, ref objectContext, ref outputFooter); } + /// + /// Sets the expiration time for the specified key and fields in a sorted set. + /// + /// The type of the object context. + /// The key of the sorted set. + /// The members within the sorted set to set the expiration time for. + /// The expiration time as a DateTimeOffset. + /// The expiration option to use. + /// The results of the operation, indicating the number of fields that were successfully set to expire. + /// The context of the object store. + /// Returns a GarnetStatus indicating the success or failure of the operation. + public GarnetStatus SortedSetExpire(ArgSlice key, ReadOnlySpan members, DateTimeOffset expireAt, ExpireOption expireOption, out int[] results, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext + { + results = default; + var expireAtUtc = expireAt.UtcTicks; + var expiryLength = NumUtils.CountDigits(expireAtUtc); + var expirySlice = scratchBufferManager.CreateArgSlice(expiryLength); + var expirySpan = expirySlice.Span; + NumUtils.WriteInt64(expireAtUtc, expirySpan); + + parseState.Initialize(1 + members.Length); + parseState.SetArgument(0, expirySlice); + parseState.SetArguments(1, members); + + var header = new RespInputHeader(GarnetObjectType.SortedSet) { SortedSetOp = SortedSetOperation.ZEXPIRE }; + var innerInput = new ObjectInput(header, ref parseState, startIdx: 0, arg1: (int)expireOption); + + var outputFooter = new GarnetObjectStoreOutput { SpanByteAndMemory = new SpanByteAndMemory(null) }; + var status = RMWObjectStoreOperationWithOutput(key.ToArray(), ref innerInput, ref objectContext, ref outputFooter); + + if (status == GarnetStatus.OK) + { + results = ProcessRespIntegerArrayOutput(outputFooter, out _); + } + + return status; + } + /// /// Returns the time-to-live (TTL) of a SortedSet member. /// @@ -1529,6 +1572,37 @@ public GarnetStatus SortedSetTimeToLive(ArgSlice key, bool isMil return ReadObjectStoreOperationWithOutput(key.ToArray(), ref innerInput, ref objectContext, ref outputFooter); } + /// + /// Returns the time-to-live (TTL) of a SortedSet member. + /// + /// The type of the object context. + /// The key of the sorted set. + /// The members within the sorted set to get the TTL for. + /// The array of TimeSpan representing the TTL for each member. + /// The context of the object store. + /// Returns a GarnetStatus indicating the success or failure of the operation. + public GarnetStatus SortedSetTimeToLive(ArgSlice key, ReadOnlySpan members, out TimeSpan[] expireIn, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext + { + expireIn = default; + var header = new RespInputHeader(GarnetObjectType.SortedSet) { SortedSetOp = SortedSetOperation.ZTTL }; + parseState.Initialize(members.Length); + parseState.SetArguments(0, members); + var isMilliseconds = 1; + var isTimestamp = 0; + var innerInput = new ObjectInput(header, ref parseState, arg1: isMilliseconds, arg2: isTimestamp); + + var outputFooter = new GarnetObjectStoreOutput { SpanByteAndMemory = new SpanByteAndMemory(null) }; + var status = ReadObjectStoreOperationWithOutput(key.ToArray(), ref innerInput, ref objectContext, ref outputFooter); + + if (status == GarnetStatus.OK) + { + expireIn = ProcessRespIntegerArrayOutput(outputFooter, out _).Select(x => TimeSpan.FromMilliseconds(x < 0 ? 0 : x)).ToArray(); + } + + return status; + } + /// /// Removes the expiration time from a SortedSet member, making it persistent. /// @@ -1542,6 +1616,35 @@ public GarnetStatus SortedSetPersist(ArgSlice key, ref ObjectInp where TObjectContext : ITsavoriteContext => RMWObjectStoreOperationWithOutput(key.ToArray(), ref input, ref objectContext, ref outputFooter); + /// + /// Removes the expiration time from the specified members in the sorted set stored at the given key. + /// + /// The type of the object context. + /// The key of the sorted set. + /// The members whose expiration time will be removed. + /// The results of the operation, indicating the number of members whose expiration time was successfully removed. + /// The context of the object store. + /// Returns a GarnetStatus indicating the success or failure of the operation. + public GarnetStatus SortedSetPersist(ArgSlice key, ReadOnlySpan members, out int[] results, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext + { + results = default; + var header = new RespInputHeader(GarnetObjectType.SortedSet) { SortedSetOp = SortedSetOperation.ZPERSIST }; + parseState.Initialize(members.Length); + parseState.SetArguments(0, members); + var innerInput = new ObjectInput(header, ref parseState); + var outputFooter = new GarnetObjectStoreOutput { SpanByteAndMemory = new SpanByteAndMemory(null) }; + + var status = RMWObjectStoreOperationWithOutput(key.ToArray(), ref innerInput, ref objectContext, ref outputFooter); + + if (status == GarnetStatus.OK) + { + results = ProcessRespIntegerArrayOutput(outputFooter, out _); + } + + return status; + } + /// /// Collects SortedSet keys and performs a specified operation on them. /// @@ -1601,5 +1704,46 @@ public GarnetStatus SortedSetCollect(ReadOnlySpan keys _zcollectTaskLock.WriteUnlock(); } } + + /// + /// Collects SortedSet keys and performs a specified operation on them. + /// + /// The type of the object context. + /// The object context for the operation. + /// The status of the operation. + /// + /// If the first key is "*", all SortedSet keys are scanned in batches and the operation is performed on each key. + /// Otherwise, the operation is performed on the specified keys. + /// + public GarnetStatus SortedSetCollect(ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext + { + return SortedSetCollect([], ref objectContext); + } + + /// + /// Collects SortedSet keys and performs a specified operation on them. + /// + /// The type of the object context. + /// The keys to collect. + /// The object context for the operation. + /// The status of the operation. + /// + /// If the first key is "*", all SortedSet keys are scanned in batches and the operation is performed on each key. + /// Otherwise, the operation is performed on the specified keys. + /// + public GarnetStatus SortedSetCollect(ReadOnlySpan keys, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext + { + var header = new RespInputHeader(GarnetObjectType.SortedSet) { SortedSetOp = SortedSetOperation.ZCOLLECT }; + var innerInput = new ObjectInput(header); + + if (keys.IsEmpty) + { + return SortedSetCollect([ArgSlice.FromPinnedSpan("*"u8)], ref innerInput, ref objectContext); + } + + return SortedSetCollect(keys, ref innerInput, ref objectContext); + } } } \ No newline at end of file diff --git a/libs/server/StoreWrapper.cs b/libs/server/StoreWrapper.cs index 559ed6cf9c..fe4e79fcf9 100644 --- a/libs/server/StoreWrapper.cs +++ b/libs/server/StoreWrapper.cs @@ -486,11 +486,7 @@ static void ExecuteHashCollect(ScratchBufferManager scratchBufferManager, Storag static void ExecuteSortedSetCollect(ScratchBufferManager scratchBufferManager, StorageSession storageSession) { - var header = new RespInputHeader(GarnetObjectType.SortedSet) { SortedSetOp = SortedSetOperation.ZCOLLECT }; - var input = new ObjectInput(header); - - ReadOnlySpan key = [ArgSlice.FromPinnedSpan("*"u8)]; - storageSession.SortedSetCollect(key, ref input, ref storageSession.objectStoreBasicContext); + storageSession.SortedSetCollect(ref storageSession.objectStoreBasicContext); scratchBufferManager.Reset(); } } diff --git a/playground/CommandInfoUpdater/GarnetCommandsDocs.json b/playground/CommandInfoUpdater/GarnetCommandsDocs.json index bebf4f591a..1e564a04c2 100644 --- a/playground/CommandInfoUpdater/GarnetCommandsDocs.json +++ b/playground/CommandInfoUpdater/GarnetCommandsDocs.json @@ -858,5 +858,475 @@ "Complexity": "O(1)" } ] + }, + { + "Command": "ZCOLLECT", + "Name": "ZCOLLECT", + "Summary": "Manually trigger deletion of expired members from memory for SortedSet", + "Group": "Hash" + }, + { + "Command": "ZEXPIRE", + "Name": "ZEXPIRE", + "Summary": "Set expiry for sorted set member using relative time to expire (seconds)", + "Group": "SortedSet", + "Complexity": "O(N) where N is the number of specified members", + "Arguments": [ + { + "TypeDiscriminator": "RespCommandKeyArgument", + "Name": "KEY", + "DisplayText": "key", + "Type": "Key", + "KeySpecIndex": 0 + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "SECONDS", + "DisplayText": "seconds", + "Type": "Integer" + }, + { + "TypeDiscriminator": "RespCommandContainerArgument", + "Name": "CONDITION", + "Type": "OneOf", + "ArgumentFlags": "Optional", + "Arguments": [ + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "NX", + "DisplayText": "nx", + "Type": "PureToken", + "Token": "NX" + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "XX", + "DisplayText": "xx", + "Type": "PureToken", + "Token": "XX" + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "GT", + "DisplayText": "gt", + "Type": "PureToken", + "Token": "GT" + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "LT", + "DisplayText": "lt", + "Type": "PureToken", + "Token": "LT" + } + ] + }, + { + "TypeDiscriminator": "RespCommandContainerArgument", + "Name": "MEMBERS", + "Type": "Block", + "Token": "MEMBERS", + "Arguments": [ + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "NUMMEMBERS", + "DisplayText": "nummembers", + "Type": "Integer" + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "MEMBER", + "DisplayText": "member", + "Type": "String", + "ArgumentFlags": "Multiple" + } + ] + } + ] + }, + { + "Command": "ZEXPIREAT", + "Name": "ZEXPIREAT", + "Summary": "Set expiry for sorted set member using an absolute Unix timestamp (seconds)", + "Group": "SortedSet", + "Complexity": "O(N) where N is the number of specified members", + "Arguments": [ + { + "TypeDiscriminator": "RespCommandKeyArgument", + "Name": "KEY", + "DisplayText": "key", + "Type": "Key", + "KeySpecIndex": 0 + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "UNIX-TIME-SECONDS", + "DisplayText": "unix-time-seconds", + "Type": "UnixTime" + }, + { + "TypeDiscriminator": "RespCommandContainerArgument", + "Name": "CONDITION", + "Type": "OneOf", + "ArgumentFlags": "Optional", + "Arguments": [ + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "NX", + "DisplayText": "nx", + "Type": "PureToken", + "Token": "NX" + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "XX", + "DisplayText": "xx", + "Type": "PureToken", + "Token": "XX" + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "GT", + "DisplayText": "gt", + "Type": "PureToken", + "Token": "GT" + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "LT", + "DisplayText": "lt", + "Type": "PureToken", + "Token": "LT" + } + ] + }, + { + "TypeDiscriminator": "RespCommandContainerArgument", + "Name": "MEMBERS", + "Type": "Block", + "Token": "MEMBERS", + "Arguments": [ + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "NUMMEMBERS", + "DisplayText": "nummembers", + "Type": "Integer" + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "MEMBER", + "DisplayText": "member", + "Type": "String", + "ArgumentFlags": "Multiple" + } + ] + } + ] + }, + { + "Command": "ZEXPIRETIME", + "Name": "ZEXPIRETIME", + "Summary": "Returns the expiration time of a sorted set member as a Unix timestamp, in seconds.", + "Group": "SortedSet", + "Complexity": "O(N) where N is the number of specified members", + "Arguments": [ + { + "TypeDiscriminator": "RespCommandKeyArgument", + "Name": "KEY", + "DisplayText": "key", + "Type": "Key", + "KeySpecIndex": 0 + }, + { + "TypeDiscriminator": "RespCommandContainerArgument", + "Name": "FIELDS", + "Type": "Block", + "Token": "FIELDS", + "Arguments": [ + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "NUMFIELDS", + "DisplayText": "numfields", + "Type": "Integer" + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "FIELD", + "DisplayText": "field", + "Type": "String", + "ArgumentFlags": "Multiple" + } + ] + } + ] + }, + { + "Command": "ZPERSIST", + "Name": "ZPERSIST", + "Summary": "Removes the expiration time for each specified sorted set member", + "Group": "SortedSet", + "Complexity": "O(N) where N is the number of specified members", + "Arguments": [ + { + "TypeDiscriminator": "RespCommandKeyArgument", + "Name": "KEY", + "DisplayText": "key", + "Type": "Key", + "KeySpecIndex": 0 + }, + { + "TypeDiscriminator": "RespCommandContainerArgument", + "Name": "MEMBERS", + "Type": "Block", + "Token": "MEMBERS", + "Arguments": [ + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "NUMMEMBERS", + "DisplayText": "nummembers", + "Type": "Integer" + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "MEMBER", + "DisplayText": "member", + "Type": "String", + "ArgumentFlags": "Multiple" + } + ] + } + ] + }, + { + "Command": "ZPEXPIRE", + "Name": "ZPEXPIRE", + "Summary": "Set expiry for sorted set member using relative time to expire (milliseconds)", + "Group": "SortedSet", + "Complexity": "O(N) where N is the number of specified members", + "Arguments": [ + { + "TypeDiscriminator": "RespCommandKeyArgument", + "Name": "KEY", + "DisplayText": "key", + "Type": "Key", + "KeySpecIndex": 0 + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "MILLISECONDS", + "DisplayText": "milliseconds", + "Type": "Integer" + }, + { + "TypeDiscriminator": "RespCommandContainerArgument", + "Name": "CONDITION", + "Type": "OneOf", + "ArgumentFlags": "Optional", + "Arguments": [ + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "NX", + "DisplayText": "nx", + "Type": "PureToken", + "Token": "NX" + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "XX", + "DisplayText": "xx", + "Type": "PureToken", + "Token": "XX" + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "GT", + "DisplayText": "gt", + "Type": "PureToken", + "Token": "GT" + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "LT", + "DisplayText": "lt", + "Type": "PureToken", + "Token": "LT" + } + ] + }, + { + "TypeDiscriminator": "RespCommandContainerArgument", + "Name": "MEMBERS", + "Type": "Block", + "Token": "MEMBERS", + "Arguments": [ + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "NUMMEMBERS", + "DisplayText": "nummembers", + "Type": "Integer" + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "MEMBER", + "DisplayText": "member", + "Type": "String", + "ArgumentFlags": "Multiple" + } + ] + } + ] + }, + { + "Command": "ZPEXPIREAT", + "Name": "ZPEXPIREAT", + "Summary": "Set expiry for sorted set member using an absolute Unix timestamp (milliseconds)", + "Group": "SortedSet", + "Complexity": "O(N) where N is the number of specified members", + "Arguments": [ + { + "TypeDiscriminator": "RespCommandKeyArgument", + "Name": "KEY", + "DisplayText": "key", + "Type": "Key", + "KeySpecIndex": 0 + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "UNIX-TIME-MILLISECONDS", + "DisplayText": "unix-time-milliseconds", + "Type": "UnixTime" + }, + { + "TypeDiscriminator": "RespCommandContainerArgument", + "Name": "CONDITION", + "Type": "OneOf", + "ArgumentFlags": "Optional", + "Arguments": [ + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "NX", + "DisplayText": "nx", + "Type": "PureToken", + "Token": "NX" + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "XX", + "DisplayText": "xx", + "Type": "PureToken", + "Token": "XX" + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "GT", + "DisplayText": "gt", + "Type": "PureToken", + "Token": "GT" + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "LT", + "DisplayText": "lt", + "Type": "PureToken", + "Token": "LT" + } + ] + }, + { + "TypeDiscriminator": "RespCommandContainerArgument", + "Name": "MEMBERS", + "Type": "Block", + "Token": "MEMBERS", + "Arguments": [ + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "NUMMEMBERS", + "DisplayText": "nummembers", + "Type": "Integer" + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "MEMBER", + "DisplayText": "member", + "Type": "String", + "ArgumentFlags": "Multiple" + } + ] + } + ] + }, + { + "Command": "ZPEXPIRETIME", + "Name": "ZPEXPIRETIME", + "Summary": "Returns the expiration time of a sorted set member as a Unix timestamp, in msec.", + "Group": "SortedSet", + "Complexity": "O(N) where N is the number of specified members", + "Arguments": [ + { + "TypeDiscriminator": "RespCommandKeyArgument", + "Name": "KEY", + "DisplayText": "key", + "Type": "Key", + "KeySpecIndex": 0 + }, + { + "TypeDiscriminator": "RespCommandContainerArgument", + "Name": "MEMBERS", + "Type": "Block", + "Token": "MEMBERS", + "Arguments": [ + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "NUMMEMBERS", + "DisplayText": "nummembers", + "Type": "Integer" + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "MEMBER", + "DisplayText": "member", + "Type": "String", + "ArgumentFlags": "Multiple" + } + ] + } + ] + }, + { + "Command": "ZPTTL", + "Name": "ZPTTL", + "Summary": "Returns the TTL in milliseconds of a sorted set member.", + "Group": "SortedSet", + "Complexity": "O(N) where N is the number of specified members", + "Arguments": [ + { + "TypeDiscriminator": "RespCommandKeyArgument", + "Name": "KEY", + "DisplayText": "key", + "Type": "Key", + "KeySpecIndex": 0 + }, + { + "TypeDiscriminator": "RespCommandContainerArgument", + "Name": "MEMBERS", + "Type": "Block", + "Token": "MEMBERS", + "Arguments": [ + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "NUMMEMBERS", + "DisplayText": "nummembers", + "Type": "Integer" + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "MEMBER", + "DisplayText": "member", + "Type": "String", + "ArgumentFlags": "Multiple" + } + ] + } + ] } ] \ No newline at end of file diff --git a/test/Garnet.test/TestProcedureSortedSets.cs b/test/Garnet.test/TestProcedureSortedSets.cs index 1a905a2699..c249d8ae91 100644 --- a/test/Garnet.test/TestProcedureSortedSets.cs +++ b/test/Garnet.test/TestProcedureSortedSets.cs @@ -114,6 +114,22 @@ private static bool TestAPI(TGarnetApi api, ref CustomProcedureInput if (status != GarnetStatus.OK || newScore != 12345) return false; + status = api.SortedSetExpire(ssA, [.. ssItems.Skip(4).Take(1).Select(x => x.member), ArgSlice.FromPinnedSpan(Encoding.UTF8.GetBytes("nonExist"))], DateTimeOffset.UtcNow.AddMinutes(10), ExpireOption.None, out var expireResults); + if (status != GarnetStatus.OK || expireResults.Length != 2 || expireResults[0] != 1 || expireResults[1] != -2) + return false; + + status = api.SortedSetTimeToLive(ssA, [.. ssItems.Skip(4).Take(1).Select(x => x.member), ArgSlice.FromPinnedSpan(Encoding.UTF8.GetBytes("nonExist"))], out var expireIn); + if (status != GarnetStatus.OK || expireIn.Length != 2 || expireIn[0].TotalMicroseconds == 0 || expireIn[1].TotalMicroseconds != 0) + return false; + + status = api.SortedSetPersist(ssA, [.. ssItems.Skip(4).Take(1).Select(x => x.member), ArgSlice.FromPinnedSpan(Encoding.UTF8.GetBytes("nonExist"))], out var persistResults); + if (status != GarnetStatus.OK || persistResults.Length != 2 || persistResults[0] != 1 || persistResults[1] != -2) + return false; + + status = api.SortedSetCollect([ssA]); + if (status != GarnetStatus.OK) + return false; + // Exercise SortedSetRemoveRangeByScore status = api.SortedSetRemoveRangeByScore(ssA, "12345", "12345", out var countRemoved); if (status != GarnetStatus.OK || countRemoved != 1)