File size: 4,172 Bytes
6993919 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 | #!/usr/bin/env python3
"""Refresh TTL on existing RAG keys to match new persistence policy."""
import subprocess
import sys
def redis_cli(*args):
result = subprocess.run(
["docker", "exec", "rmi-redis", "redis-cli", "-a", REDIS_PASS, *args],
capture_output=True,
text=True,
)
return result.stdout.strip()
def redis_eval(lua_script, numkeys, *keys_and_args):
"""Run a Lua script via EVAL. Returns stdout."""
all_args = ["EVAL", lua_script, str(numkeys), *list(keys_and_args)]
return redis_cli(*all_args)
# Get Redis password from container env
env_out = subprocess.run(["docker", "exec", "rmi-backend", "env"], capture_output=True, text=True).stdout
REDIS_PASS = ""
for line in env_out.splitlines():
if line.startswith("REDIS_PASSWORD="):
REDIS_PASS = line.split("=", 1)[1]
break
if not REDIS_PASS:
print("ERROR: Could not find REDIS_PASSWORD")
sys.exit(1)
# TTL policy: 0 = persist (no expiry), N = seconds
TTL_POLICY = {
"scam_patterns": 0, # permanent
"contract_audits": 0, # permanent
"transaction_patterns": 0, # permanent
"forensic_reports": 0, # permanent
"known_scams": 0, # permanent (was 365d)
"wallet_profiles": 86400 * 365, # 1 year
"market_intel": 86400 * 365, # 1 year
"token_analysis": 86400 * 90, # 90 days
"news_articles": 86400 * 30, # 30 days
"general": 86400 * 30, # 30 days
}
total_scanned = 0
total_persisted = 0
total_refreshed = 0
errors = 0
# Lua script that processes keys in batches
# For PERSIST (ttl=0): only persist keys that have a TTL (not -1)
# For EXPIRE (ttl>0): only set expiry on keys that are persistent (-1) or have shorter TTL
LUA_BATCH = """
local collection = ARGV[1]
local desired_ttl = tonumber(ARGV[2])
local count = 0
local persist_count = 0
local refresh_count = 0
-- Use SCAN to iterate keys
local cursor = '0'
repeat
local reply = redis.call('SCAN', cursor, 'MATCH', 'rag:' .. collection .. ':*', 'COUNT', 500)
cursor = reply[1]
local keys = reply[2]
for i, key in ipairs(keys) do
count = count + 1
local current_ttl = redis.call('TTL', key)
if desired_ttl == 0 then
-- Make permanent: only if not already persistent (-1)
if current_ttl ~= -1 then
redis.call('PERSIST', key)
persist_count = persist_count + 1
end
else
-- Set expiry if currently persistent (-1) or TTL is shorter than desired
if current_ttl == -1 then
-- Already persistent, leave it (don't downgrade)
elseif current_ttl < desired_ttl then
redis.call('EXPIRE', key, desired_ttl)
refresh_count = refresh_count + 1
end
end
end
until cursor == '0'
return {count, persist_count, refresh_count}
"""
print("Refreshing RAG TTLs according to persistence policy...")
print()
for collection, desired_ttl in TTL_POLICY.items():
result = redis_eval(LUA_BATCH, 0, collection, str(desired_ttl))
# Result format: one integer per line (Lua array serializes as newline-separated)
lines = [l.strip() for l in result.splitlines() if l.strip()]
if len(lines) == 3:
try:
scanned = int(lines[0].lstrip("[").rstrip(","))
persisted = int(lines[1].rstrip(","))
refreshed = int(lines[2].rstrip("]"))
except (ValueError, IndexError):
print(f" {collection}: unexpected result: {result}")
errors += 1
continue
action = f"TTL={0} (persistent)" if desired_ttl == 0 else f"TTL={desired_ttl}s ({desired_ttl // 86400}d)"
print(f" {collection}: {scanned} keys scanned, {persisted} persisted, {refreshed} refreshed [{action}]")
total_scanned += scanned
total_persisted += persisted
total_refreshed += refreshed
else:
print(f" {collection}: unexpected result: {result}")
errors += 1
print()
print(
f"Done: {total_scanned} keys scanned, {total_persisted} persisted (TTL removed), {total_refreshed} TTLs refreshed"
)
if errors:
print(f"Errors: {errors}")
|