import express from "express"; import { fileURLToPath } from "url"; import { dirname, join } from "path"; const __filename = fileURLToPath(import.meta.url); const __dirname = dirname(__filename); const app = express(); const PORT = process.env.PORT || 7860; const WFM_BASE = "https://api.warframe.market/v2"; const WFM_HEADERS = { Platform: "pc", Language: "en" }; // --- Rate limiter: token bucket, 5 tokens/sec burst-friendly --- class RateLimiter { constructor(maxPerSecond = 5) { this.maxPerSecond = maxPerSecond; this.tokens = maxPerSecond; this.lastRefill = Date.now(); } async acquire() { const now = Date.now(); const elapsed = now - this.lastRefill; if (elapsed >= 1000) { this.tokens = Math.min(this.maxPerSecond, this.tokens + Math.floor(elapsed / 1000) * this.maxPerSecond); this.lastRefill = now; } if (this.tokens <= 0) { const wait = 1000 - elapsed; await new Promise((r) => setTimeout(r, wait)); this.tokens = this.maxPerSecond; this.lastRefill = Date.now(); } this.tokens--; } } const limiter = new RateLimiter(15); // --- In-memory cache --- class Cache { constructor(ttlMs) { this.ttlMs = ttlMs; this.store = new Map(); } get(key) { const entry = this.store.get(key); if (!entry) return null; if (Date.now() - entry.ts > this.ttlMs) { this.store.delete(key); return null; } return entry.data; } set(key, data) { this.store.set(key, { data, ts: Date.now() }); } invalidate(key) { this.store.delete(key); } clear() { this.store.clear(); } } const catalogCache = new Cache(60 * 60 * 1000); // 1 hour const ordersCache = new Cache(60 * 1000); // 60s const priceCache = new Cache(60 * 1000); // 60s for batch price lookups // --- Upstream fetch with rate limiting --- async function wfmFetch(path) { await limiter.acquire(); const res = await fetch(`${WFM_BASE}${path}`, { headers: WFM_HEADERS }); if (!res.ok) { const text = await res.text(); throw new Error(`WFM API ${res.status}: ${text}`); } return res.json(); } // --- API routes --- app.get("/api/items", async (req, res) => { try { let items = catalogCache.get("catalog"); if (!items) { const body = await wfmFetch("/items"); items = body.data; catalogCache.set("catalog", items); } res.json(items); } catch (err) { console.error("Error fetching items:", err.message); res.status(502).json({ error: "Failed to fetch item catalog from Warframe Market" }); } }); app.get("/api/refresh-catalog", async (req, res) => { try { catalogCache.clear(); const body = await wfmFetch("/items"); catalogCache.set("catalog", body.data); res.json({ ok: true, count: body.data.length }); } catch (err) { console.error("Error refreshing catalog:", err.message); res.status(502).json({ error: "Failed to refresh catalog" }); } }); app.get("/api/orders/:slug", async (req, res) => { const { slug } = req.params; try { let orders = ordersCache.get(slug); if (!orders) { const body = await wfmFetch(`/orders/item/${encodeURIComponent(slug)}`); orders = body.data; ordersCache.set(slug, orders); } res.json(orders); } catch (err) { console.error(`Error fetching orders for ${slug}:`, err.message); res.status(502).json({ error: `Failed to fetch orders for ${slug}` }); } }); // --- Batch price lookup --- app.use(express.json()); app.post("/api/prices", async (req, res) => { const { slugs } = req.body; if (!Array.isArray(slugs) || slugs.length === 0) { return res.status(400).json({ error: "slugs array required" }); } const result = {}; const toFetch = []; for (const slug of slugs) { const cached = priceCache.get(slug); if (cached !== null) { result[slug] = cached; } else { toFetch.push(slug); } } // Build a slug→tags lookup from catalog cache (ensure it's loaded) let catalog = catalogCache.get("catalog"); if (!catalog) { try { const body = await wfmFetch("/items"); catalog = body.data; catalogCache.set("catalog", catalog); } catch { catalog = []; } } const slugTags = {}; for (const item of catalog) { slugTags[item.slug] = item.tags || []; } // Fetch missing prices in parallel batches const CONCURRENCY = 15; for (let i = 0; i < toFetch.length; i += CONCURRENCY) { const batch = toFetch.slice(i, i + CONCURRENCY); const results = await Promise.allSettled( batch.map(async (slug) => { const tags = slugTags[slug] || []; const isRanked = tags.includes("arcane_enhancement") || tags.includes("mod"); if (isRanked) { // For arcanes/mods, fetch all orders and filter to max-rank only const body = await wfmFetch(`/orders/item/${encodeURIComponent(slug)}`); const orders = body.data; const online = (orders || []).filter( (o) => o.user?.status === "ingame" || o.user?.status === "online" ); const maxRank = online.length > 0 ? Math.max(...online.map((o) => o.rank ?? 0)) : 0; const maxedOnline = online.filter((o) => (o.rank ?? 0) === maxRank); const sells = maxedOnline.filter((o) => o.type === "sell"); const buys = maxedOnline.filter((o) => o.type === "buy"); const lowestSell = sells.length > 0 ? Math.min(...sells.map((o) => o.platinum)) : null; const highestBuy = buys.length > 0 ? Math.max(...buys.map((o) => o.platinum)) : null; return { slug, lowestSell, highestBuy, activeSells: sells.length }; } // Normal items: use the lightweight /top endpoint const body = await wfmFetch(`/orders/item/${encodeURIComponent(slug)}/top`); const data = body.data; const sells = (data.sell || []) .filter((o) => o.user?.status === "ingame" || o.user?.status === "online"); const buys = (data.buy || []) .filter((o) => o.user?.status === "ingame" || o.user?.status === "online"); const lowestSell = sells.length > 0 ? Math.min(...sells.map((o) => o.platinum)) : null; const highestBuy = buys.length > 0 ? Math.max(...buys.map((o) => o.platinum)) : null; return { slug, lowestSell, highestBuy, activeSells: sells.length }; }) ); for (const r of results) { if (r.status === "fulfilled") { const { slug, ...prices } = r.value; result[slug] = prices; priceCache.set(slug, prices); } } } res.json(result); }); // --- SSE batch price streaming --- app.post("/api/prices/stream", async (req, res) => { const { slugs } = req.body; if (!Array.isArray(slugs) || slugs.length === 0) { return res.status(400).json({ error: "slugs array required" }); } res.setHeader("Content-Type", "text/event-stream"); res.setHeader("Cache-Control", "no-cache"); res.setHeader("Connection", "keep-alive"); res.flushHeaders(); const toFetch = []; const cached = {}; for (const slug of slugs) { const c = priceCache.get(slug); if (c !== null) { cached[slug] = c; } else { toFetch.push(slug); } } // Send cached results immediately if (Object.keys(cached).length > 0) { res.write(`data: ${JSON.stringify(cached)}\n\n`); } // Ensure catalog is loaded let catalog = catalogCache.get("catalog"); if (!catalog) { try { const body = await wfmFetch("/items"); catalog = body.data; catalogCache.set("catalog", catalog); } catch { catalog = []; } } const slugTags = {}; for (const item of catalog) { slugTags[item.slug] = item.tags || []; } // Stream results — fetch in parallel batches with retry for failures const CONCURRENCY = 10; const heartbeat = setInterval(() => res.write(": heartbeat\n\n"), 15000); async function fetchSlug(slug) { const tags = slugTags[slug] || []; const isRanked = tags.includes("arcane_enhancement") || tags.includes("mod"); for (let attempt = 0; attempt < 3; attempt++) { try { if (isRanked) { const body = await wfmFetch(`/orders/item/${encodeURIComponent(slug)}`); const orders = body.data; const online = (orders || []).filter( (o) => o.user?.status === "ingame" || o.user?.status === "online" ); const maxRank = online.length > 0 ? Math.max(...online.map((o) => o.rank ?? 0)) : 0; const maxedOnline = online.filter((o) => (o.rank ?? 0) === maxRank); const sells = maxedOnline.filter((o) => o.type === "sell"); const buys = maxedOnline.filter((o) => o.type === "buy"); const lowestSell = sells.length > 0 ? Math.min(...sells.map((o) => o.platinum)) : null; const highestBuy = buys.length > 0 ? Math.max(...buys.map((o) => o.platinum)) : null; return { slug, lowestSell, highestBuy, activeSells: sells.length }; } const body = await wfmFetch(`/orders/item/${encodeURIComponent(slug)}/top`); const data = body.data; const sells = (data.sell || []) .filter((o) => o.user?.status === "ingame" || o.user?.status === "online"); const buys = (data.buy || []) .filter((o) => o.user?.status === "ingame" || o.user?.status === "online"); const lowestSell = sells.length > 0 ? Math.min(...sells.map((o) => o.platinum)) : null; const highestBuy = buys.length > 0 ? Math.max(...buys.map((o) => o.platinum)) : null; return { slug, lowestSell, highestBuy, activeSells: sells.length }; } catch { if (attempt < 2) await new Promise((r) => setTimeout(r, 500 * (attempt + 1))); } } return null; // failed after 3 attempts } const failedSlugs = []; for (let i = 0; i < toFetch.length; i += CONCURRENCY) { const batch = toFetch.slice(i, i + CONCURRENCY); const results = await Promise.all(batch.map((slug) => fetchSlug(slug))); const batchResult = {}; for (const r of results) { if (r) { const { slug, ...prices } = r; batchResult[slug] = prices; priceCache.set(slug, prices); } } // Track failed slugs for a final retry pass for (let j = 0; j < results.length; j++) { if (!results[j]) failedSlugs.push(batch[j]); } if (Object.keys(batchResult).length > 0) { res.write(`data: ${JSON.stringify(batchResult)}\n\n`); } } // Final retry pass for any items that failed if (failedSlugs.length > 0) { const retryResult = {}; for (const slug of failedSlugs) { const r = await fetchSlug(slug); if (r) { const { slug: s, ...prices } = r; retryResult[s] = prices; priceCache.set(s, prices); } } if (Object.keys(retryResult).length > 0) { res.write(`data: ${JSON.stringify(retryResult)}\n\n`); } } clearInterval(heartbeat); res.write("event: done\ndata: {}\n\n"); res.end(); }); // --- Serve static React build --- const clientDist = join(__dirname, "..", "client", "dist"); app.use(express.static(clientDist)); // SPA fallback app.get("*", (req, res) => { res.sendFile(join(clientDist, "index.html")); }); app.listen(PORT, "0.0.0.0", () => { console.log(`WFM Tracker server listening on port ${PORT}`); });