diff --git a/CLAUDE.md b/CLAUDE.md index 82033eeb..5013f4a5 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -109,8 +109,8 @@ Optional: - `GITHUB_CALLBACK_URL` - OAuth callback URL (default: `/auth/github/callback`) - `WEB_SESSION_TTL_HOURS` - How long a minted `storageState` session stays fresh before re-minting (default: 1) - `WEB_SESSION_MINT_TIMEOUT_SECONDS` - Max time to wait for a broker mint before failing the request (default: 180) -- `GEOIP_DB_DIR` - Directory holding GeoLite2-City.mmdb + GeoLite2-ASN.mmdb (default: `./geoip`). Absent DBs = all non-public agents stay Unverified (safe default; local dev needs no MaxMind account) -- `MAXMIND_LICENSE_KEY` - Free GeoLite2 license key (signup artifact, never a payment), consumed by `scripts/geoip-refresh.sh` (run weekly) — not read by the server itself. When unset, the script automatically falls back to DB-IP Lite (CC-BY-4.0, no account/key; slightly less accurate, no accuracy_radius); force with `--source dbip` +- `GEOIP_DB_DIR` - Directory holding the GeoIP databases (canonical `City.mmdb`/`ASN.mmdb`, falling back to legacy `GeoLite2-City.mmdb`/`GeoLite2-ASN.mmdb` names if present) (default: `./geoip`). Absent DBs = all non-public agents stay Unverified (safe default; local dev needs no MaxMind account) — self-heals on next startup/refresh +- `MAXMIND_LICENSE_KEY` - Optional bootstrap-only GeoLite2 license key (signup artifact, never a payment). The MaxMind key is normally managed from the admin console (Regions page → GeoIP databases card), stored encrypted in `systemConfig` (requires `CREDENTIAL_ENCRYPTION_KEY`); this env var is only consulted as a fallback when no console-managed key is set. No cron needed — Coolify containers have ephemeral filesystems, so `server/geoip-refresh.ts` refreshes in-app instead of via a script: at startup when the DBs are missing/stale (>7 days), on a weekly in-process timer, and on demand via the admin Refresh button / `POST /api/admin/geoip/refresh`. With no key configured (console or env), it automatically falls back to DB-IP Lite (CC-BY-4.0, no account/key; slightly less accurate, no accuracy_radius) Auth-session broker sidecar (registration env is broker-only, not read by Core — see Auth-Session Broker below): - `VOX_CORE_URL` - Core base URL the broker registers/heartbeats against diff --git a/client/src/pages/admin-regions.tsx b/client/src/pages/admin-regions.tsx index dd85342d..f3a36c8c 100644 --- a/client/src/pages/admin-regions.tsx +++ b/client/src/pages/admin-regions.tsx @@ -1,8 +1,9 @@ import { useState } from "react"; -import { useMutation } from "@tanstack/react-query"; -import { MapPin, Pencil, Plus, Trash2 } from "lucide-react"; +import { useMutation, useQuery } from "@tanstack/react-query"; +import { MapPin, Pencil, Plus, Trash2, RefreshCw, Database, ExternalLink } from "lucide-react"; import { Badge } from "@/components/ui/badge"; import { Button } from "@/components/ui/button"; +import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card"; import { Dialog, DialogContent, DialogDescription, DialogFooter, DialogHeader, DialogTitle } from "@/components/ui/dialog"; import { Input } from "@/components/ui/input"; import { Label } from "@/components/ui/label"; @@ -14,6 +15,32 @@ import { useRegionLocations } from "@/hooks/use-regions"; import { apiRequest, queryClient } from "@/lib/queryClient"; import type { RegionLocation } from "@/lib/utils"; +type GeoipDatabaseInfo = { + name: "City" | "ASN"; + present: boolean; + sizeBytes: number | null; + modifiedAt: string | null; +}; + +type GeoipStatus = { + source: "geolite2" | "dbip"; + dir: string; + state: "idle" | "refreshing"; + databases: GeoipDatabaseInfo[]; + lastRefresh: { ok: boolean; source: string; at: string; error?: string } | null; + attribution: string | null; + maxmindKey: { configured: boolean; source: "console" | "env" | null }; +}; + +function fmtBytes(bytes: number | null): string { + if (bytes === null) return "—"; + return `${(bytes / (1024 * 1024)).toFixed(1)} MB`; +} + +function fmtWhen(value: string | null | undefined): string { + return value ? new Date(value).toLocaleString() : "—"; +} + type RegionForm = { baseId: string; displayName: string; @@ -41,6 +68,46 @@ export default function AdminRegions() { const [editing, setEditing] = useState(null); const [form, setForm] = useState(EMPTY_FORM); + const { data: geoipStatus, isLoading: geoipLoading } = useQuery({ + queryKey: ["/api/admin/geoip/status"], + refetchInterval: (query) => (query.state.data?.state === "refreshing" ? 2000 : false), + }); + const [maxmindKeyInput, setMaxmindKeyInput] = useState(""); + + const refreshGeoipMutation = useMutation({ + mutationFn: async () => apiRequest("POST", "/api/admin/geoip/refresh"), + onSuccess: () => { + queryClient.invalidateQueries({ queryKey: ["/api/admin/geoip/status"] }); + toast({ title: "GeoIP refresh started" }); + }, + onError: (error: Error) => { + toast({ title: "Could not start refresh", description: error.message, variant: "destructive" }); + }, + }); + + const saveMaxmindKeyMutation = useMutation({ + mutationFn: async (key: string) => apiRequest("PUT", "/api/admin/geoip/maxmind-key", { key }), + onSuccess: () => { + setMaxmindKeyInput(""); + queryClient.invalidateQueries({ queryKey: ["/api/admin/geoip/status"] }); + toast({ title: "MaxMind key saved", description: "A GeoIP refresh has been triggered." }); + }, + onError: (error: Error) => { + toast({ title: "Could not save MaxMind key", description: error.message, variant: "destructive" }); + }, + }); + + const clearMaxmindKeyMutation = useMutation({ + mutationFn: async () => apiRequest("DELETE", "/api/admin/geoip/maxmind-key"), + onSuccess: () => { + queryClient.invalidateQueries({ queryKey: ["/api/admin/geoip/status"] }); + toast({ title: "MaxMind key cleared" }); + }, + onError: (error: Error) => { + toast({ title: "Could not clear MaxMind key", description: error.message, variant: "destructive" }); + }, + }); + const refresh = () => { queryClient.invalidateQueries({ queryKey: ["/api/admin/region-locations"] }); queryClient.invalidateQueries({ queryKey: ["/api/region-locations"] }); @@ -130,6 +197,115 @@ export default function AdminRegions() { + + + GeoIP databases + + City + ASN databases used for agent region detection. Downloaded from MaxMind GeoLite2 when a license + key is configured, otherwise from DB-IP Lite. + + + + {geoipLoading || !geoipStatus ? ( + + ) : ( + <> +
+ + {geoipStatus.source === "geolite2" ? "GeoLite2" : "DB-IP Lite"} + + {geoipStatus.state === "refreshing" && ( + Refreshing… + )} + {geoipStatus.lastRefresh && ( + + Last refresh: {geoipStatus.lastRefresh.ok ? "success" : "failed"} at {fmtWhen(geoipStatus.lastRefresh.at)} + {!geoipStatus.lastRefresh.ok && geoipStatus.lastRefresh.error ? ` — ${geoipStatus.lastRefresh.error}` : ""} + + )} + +
+ +
+ {geoipStatus.databases.map((db) => ( +
+ {db.name} + {db.present ? ( + {fmtBytes(db.sizeBytes)} · {fmtWhen(db.modifiedAt)} + ) : ( + Missing + )} +
+ ))} +
+ + {geoipStatus.source === "dbip" && geoipStatus.attribution && ( +

+ {geoipStatus.attribution} —{" "} + + db-ip.com + +

+ )} + +
+ + {geoipStatus.maxmindKey.configured && geoipStatus.maxmindKey.source === "console" ? ( +
+ Configured (console) + +
+ ) : ( +
+ {geoipStatus.maxmindKey.configured && geoipStatus.maxmindKey.source === "env" && ( + Configured (env) + )} + setMaxmindKeyInput(e.target.value)} + className="sm:max-w-xs" + /> + +
+ )} +

+ Saving a key immediately triggers a refresh and switches the source to GeoLite2. Removing it falls + back to the env var (if set) or DB-IP Lite. +

+
+ + )} + + +
diff --git a/scripts/geoip-refresh.sh b/scripts/geoip-refresh.sh deleted file mode 100755 index ca187644..00000000 --- a/scripts/geoip-refresh.sh +++ /dev/null @@ -1,86 +0,0 @@ -#!/usr/bin/env bash -# Download/refresh GeoIP databases for the zero-trust agent-region pipeline. -# Run weekly (cron / Coolify scheduled command). Missing DBs are non-fatal at -# runtime: agents simply stay Unverified. -# -# Two sources, both free: -# maxmind (default when MAXMIND_LICENSE_KEY is set) — GeoLite2, best -# accuracy; the key is a free-signup artifact, never a payment. -# dbip (fallback when the key is unset, or forced via --source dbip) — -# DB-IP Lite (CC-BY-4.0), no account or key needed; slightly less -# accurate and lacks accuracy_radius. Files are saved under the server's -# expected GeoLite2-*.mmdb names (server/location.ts loads those paths; -# the MMDB format is identical). -set -euo pipefail - -GEOIP_DB_DIR="${GEOIP_DB_DIR:-$(pwd)/geoip}" -mkdir -p "$GEOIP_DB_DIR" - -SOURCE="auto" -if [ "${1:-}" = "--source" ] && [ -n "${2:-}" ]; then - SOURCE="$2" -fi -if [ "$SOURCE" = "auto" ]; then - if [ -n "${MAXMIND_LICENSE_KEY:-}" ]; then SOURCE="maxmind"; else SOURCE="dbip"; fi -fi - -verify_mmdb() { - test -s "${GEOIP_DB_DIR}/$1.mmdb" || { echo "ERROR: $1.mmdb missing/empty after extraction"; exit 1; } - echo " → ${GEOIP_DB_DIR}/$1.mmdb" -} - -# CC-BY-4.0 requires visible credit when DB-IP data is in the product. The -# marker file tells the server which source the mmdbs came from; the server -# exposes the attribution via /api/config and the public footer renders it. -# GeoLite2's EULA needs no public credit for server-side lookups → no marker. -ATTRIBUTION_FILE="${GEOIP_DB_DIR}/ATTRIBUTION" -DBIP_ATTRIBUTION="This product includes IP geolocation data created by DB-IP, available from https://db-ip.com" - -fetch_maxmind() { - : "${MAXMIND_LICENSE_KEY:?Set MAXMIND_LICENSE_KEY (https://www.maxmind.com → GeoLite2 free account) or use --source dbip}" - rm -f "$ATTRIBUTION_FILE" - for edition in GeoLite2-City GeoLite2-ASN; do - echo "Fetching ${edition} (MaxMind GeoLite2)..." - tmp=$(mktemp -d) - curl -fsSL "https://download.maxmind.com/app/geoip_download?edition_id=${edition}&license_key=${MAXMIND_LICENSE_KEY}&suffix=tar.gz" \ - -o "${tmp}/${edition}.tar.gz" - tar -xzf "${tmp}/${edition}.tar.gz" -C "$tmp" - find "$tmp" -name "${edition}.mmdb" -exec mv {} "${GEOIP_DB_DIR}/" \; - rm -rf "$tmp" - verify_mmdb "${edition}" - done -} - -# DB-IP publishes one file per month (dbip-city-lite-YYYY-MM.mmdb.gz). Early in -# a month the current file can 404 before publication, so fall back one month. -fetch_dbip_edition() { - local dbip_name="$1" target="$2" month prev - month=$(date -u +%Y-%m) - prev=$(date -u -d "$(date -u +%Y-%m-01) -1 month" +%Y-%m 2>/dev/null || date -u -v-1m +%Y-%m) - tmp=$(mktemp -d) - echo "Fetching ${dbip_name} (DB-IP Lite, no key)..." - if ! curl -fsSL "https://download.db-ip.com/free/${dbip_name}-${month}.mmdb.gz" -o "${tmp}/db.mmdb.gz"; then - echo " ${month} not published yet, trying ${prev}..." - curl -fsSL "https://download.db-ip.com/free/${dbip_name}-${prev}.mmdb.gz" -o "${tmp}/db.mmdb.gz" - fi - gunzip "${tmp}/db.mmdb.gz" - mv "${tmp}/db.mmdb" "${GEOIP_DB_DIR}/${target}.mmdb" - rm -rf "$tmp" - verify_mmdb "${target}" -} - -fetch_dbip() { - fetch_dbip_edition "dbip-city-lite" "GeoLite2-City" - fetch_dbip_edition "dbip-asn-lite" "GeoLite2-ASN" - printf '%s\n' "$DBIP_ATTRIBUTION" > "$ATTRIBUTION_FILE" - echo "Note: DB-IP Lite data (CC-BY-4.0) saved under the server's expected filenames." - echo "Attribution marker written — the server will surface: ${DBIP_ATTRIBUTION}" -} - -case "$SOURCE" in - maxmind) fetch_maxmind ;; - dbip) fetch_dbip ;; - *) echo "ERROR: unknown --source '$SOURCE' (expected maxmind or dbip)"; exit 1 ;; -esac - -echo "Done. Restart the Vox server to load the new databases." diff --git a/server/geoip-refresh.ts b/server/geoip-refresh.ts new file mode 100644 index 00000000..fc2ba57b --- /dev/null +++ b/server/geoip-refresh.ts @@ -0,0 +1,347 @@ +/** + * In-app GeoIP database refresher. + * + * Coolify containers have ephemeral filesystems, so the old cron script + * (scripts/geoip-refresh.sh) could not keep GEOIP_DB_DIR populated across + * redeploys. This module downloads the same two databases the script used + * to fetch — MaxMind GeoLite2 (City + ASN) when a license key is available, + * DB-IP Lite (CC-BY-4.0, no key needed) otherwise — and is driven from three + * places: server startup (server/location.ts), a weekly timer (also owned by + * location.ts), and an admin "Refresh" button (server/routes.ts). + * + * Import direction: this module never imports server/location.ts. Its + * `reload` step (re-opening the mmdb readers after a successful download) is + * an INJECTED dependency — callers that need it (location.ts's startup/timer + * hook, and the admin refresh route) pass `reloadGeoReaders` in explicitly. + * That keeps the dependency graph one-directional (location.ts -> this file) + * instead of a cycle. + * + * The MaxMind license key can live in two places, resolved in this order by + * getMaxmindKey(): an admin-console-managed value in `systemConfig` + * (encrypted with the same AES-256-GCM primitives the secrets feature uses), + * then the MAXMIND_LICENSE_KEY env var as a bootstrap fallback. The key is + * never returned from getMaxmindKey() to a caller that might log or persist + * it in the clear beyond systemConfig, and it is composed onto a download URL + * only in memory, immediately before the request — buildDownloadUrls() never + * embeds it in a string that could end up in a log line or a thrown error. + */ +import path from "path"; +import { promises as fsp } from "fs"; +import { spawn } from "child_process"; +import { mkdtemp, rm } from "fs/promises"; +import { tmpdir } from "os"; +import { gunzip as gunzipCb } from "zlib"; +import { promisify } from "util"; +import { open as maxmindOpen } from "maxmind"; +import { storage, encryptValue, decryptValue, isEncryptionConfigured } from "./storage"; + +const gunzipAsync = promisify(gunzipCb); + +// Duplicated (not imported) from server/location.ts on purpose — see the +// import-direction note above. It is a one-line env lookup, not worth a cycle. +export const GEOIP_DIR = process.env.GEOIP_DB_DIR || path.join(process.cwd(), "geoip"); + +export const MAXMIND_KEY_CONFIG_KEY = "maxmind_license_key"; + +// City/ASN files run 50-90MB; 120s is the floor for a healthy connection to +// finish, not a target. Without this, a hung fetch would leave module `state` +// stuck at "refreshing" forever — every future POST /refresh would 409 until +// the process restarts. +export const DOWNLOAD_TIMEOUT_MS = 120_000; + +// CC BY 4.0 requires visible credit when DB-IP data ships in the product. +export const DBIP_ATTRIBUTION = "IP Geolocation by DB-IP (db-ip.com), CC BY 4.0"; + +export type GeoipSource = "geolite2" | "dbip"; +export type MaxmindKeySource = "console" | "env" | null; + +/** + * Resolves which key wins when both a console-managed value and the env var + * are present: console beats env. Used by getMaxmindKey() and by the + * admin status route so both report the same answer. + */ +export async function getMaxmindKey(): Promise<{ key: string | null; source: MaxmindKeySource }> { + try { + const row = await storage.getConfig(MAXMIND_KEY_CONFIG_KEY); + if (row?.value) { + return { key: decryptValue(row.value), source: "console" }; + } + } catch (err) { + // Decryption can fail if CREDENTIAL_ENCRYPTION_KEY rotated out from under + // a stored value — degrade to the env fallback rather than crash a + // refresh over it. + console.error( + `[geoip] console-managed MaxMind key could not be decrypted, falling back to env (${err instanceof Error ? err.message : String(err)})`, + ); + } + if (process.env.MAXMIND_LICENSE_KEY) { + return { key: process.env.MAXMIND_LICENSE_KEY, source: "env" }; + } + return { key: null, source: null }; +} + +/** Pure: the source is entirely determined by whether a key is present. */ +export function resolveGeoipSource(maxmindKey: string | null | undefined): GeoipSource { + return maxmindKey ? "geolite2" : "dbip"; +} + +export interface DownloadUrls { + city: string; + asn: string; + // Only set for dbip — the current month's file can 404 before publication. + cityFallback?: string; + asnFallback?: string; +} + +function yyyymm(d: Date): string { + return `${d.getUTCFullYear()}-${String(d.getUTCMonth() + 1).padStart(2, "0")}`; +} + +function previousMonth(d: Date): Date { + return new Date(Date.UTC(d.getUTCFullYear(), d.getUTCMonth() - 1, 1)); +} + +/** + * Pure and deterministic given `now` — never touches the network or embeds + * the MaxMind license key (that is composed separately, at fetch time, by + * refreshGeoipDatabases()). + */ +export function buildDownloadUrls(source: GeoipSource, now: Date = new Date()): DownloadUrls { + if (source === "geolite2") { + return { + city: "https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-City&suffix=tar.gz", + asn: "https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-ASN&suffix=tar.gz", + }; + } + const month = yyyymm(now); + const prev = yyyymm(previousMonth(now)); + return { + city: `https://download.db-ip.com/free/dbip-city-lite-${month}.mmdb.gz`, + cityFallback: `https://download.db-ip.com/free/dbip-city-lite-${prev}.mmdb.gz`, + asn: `https://download.db-ip.com/free/dbip-asn-lite-${month}.mmdb.gz`, + asnFallback: `https://download.db-ip.com/free/dbip-asn-lite-${prev}.mmdb.gz`, + }; +} + +function withLicenseKey(url: string, key: string): string { + return `${url}&license_key=${encodeURIComponent(key)}`; +} + +/** + * Defense in depth: strip a license_key value out of any message we log or + * persist. This is the SOLE guard against a key landing in error text that + * flows into `lastResult.error` — which GET /api/admin/geoip/status echoes + * verbatim, and whose response the request-logging middleware writes to the + * server log. Exported so tests can exercise it directly with a key-bearing + * message shaped like the real MaxMind fetch error (see + * tests/geoip-refresh.test.ts). + */ +export function sanitizeErrorMessage(msg: string): string { + return msg.replace(/license_key=[^&\s]+/gi, "license_key=REDACTED"); +} + +export interface RefreshDeps { + dir: string; + now: Date; + getMaxmindKey: () => Promise<{ key: string | null; source: MaxmindKeySource }>; + download: (url: string) => Promise; + gunzip: (data: Buffer) => Promise; + extractTarGz: (data: Buffer, editionId: string) => Promise; + writeFile: (path: string, data: Buffer | string) => Promise; + rename: (from: string, to: string) => Promise; + unlink: (path: string) => Promise; + validateMmdb: (path: string) => Promise; + reload: () => Promise; +} + +export interface RefreshResult { + ok: boolean; + source: GeoipSource; + at: string; + error?: string; +} + +interface DbTarget { + name: "City" | "ASN"; + editionId: "GeoLite2-City" | "GeoLite2-ASN"; +} +const TARGETS: DbTarget[] = [ + { name: "City", editionId: "GeoLite2-City" }, + { name: "ASN", editionId: "GeoLite2-ASN" }, +]; + +// Exported for tests only — lets tests prove the AbortSignal timeout wiring +// and its failure path without waiting out a real 120s timeout. +export async function downloadDefault(url: string): Promise { + const res = await fetch(url, { signal: AbortSignal.timeout(DOWNLOAD_TIMEOUT_MS) }); + if (!res.ok) { + const err = new Error(`HTTP ${res.status} downloading GeoIP database`) as Error & { status?: number }; + err.status = res.status; + throw err; + } + return Buffer.from(await res.arrayBuffer()); +} + +async function gunzipDefault(data: Buffer): Promise { + return gunzipAsync(data); +} + +/** + * MaxMind ships each edition as a tar.gz containing a single dated directory + * (e.g. GeoLite2-City_20260101/GeoLite2-City.mmdb). Shells out to the system + * `tar` rather than pulling in a tar-parsing dependency for one call site. + */ +async function extractTarGzDefault(data: Buffer, editionId: string): Promise { + const workDir = await mkdtemp(path.join(tmpdir(), "geoip-extract-")); + try { + const archivePath = path.join(workDir, "archive.tar.gz"); + await fsp.writeFile(archivePath, data); + await new Promise((resolve, reject) => { + const proc = spawn("tar", ["-xzf", archivePath, "-C", workDir]); + let stderr = ""; + proc.stderr.on("data", (d) => { stderr += d.toString(); }); + proc.on("error", reject); + proc.on("close", (code) => (code === 0 ? resolve() : reject(new Error(`tar exited ${code}: ${stderr.slice(0, 200)}`)))); + }); + const entries = await fsp.readdir(workDir, { withFileTypes: true }); + const extractedDir = entries.find((e) => e.isDirectory() && e.name.startsWith(editionId)); + if (!extractedDir) throw new Error(`extracted archive missing a ${editionId}_* directory`); + return await fsp.readFile(path.join(workDir, extractedDir.name, `${editionId}.mmdb`)); + } finally { + await rm(workDir, { recursive: true, force: true }); + } +} + +async function validateMmdbDefault(filePath: string): Promise { + try { + const stat = await fsp.stat(filePath); + if (stat.size <= 1024 * 1024) return false; // > 1MB — catches truncated/HTML-error downloads + await maxmindOpen(filePath); // throws if the file isn't a well-formed mmdb + return true; + } catch { + return false; + } +} + +function defaultDeps(): RefreshDeps { + return { + dir: GEOIP_DIR, + now: new Date(), + getMaxmindKey, + download: downloadDefault, + gunzip: gunzipDefault, + extractTarGz: extractTarGzDefault, + writeFile: (p, data) => fsp.writeFile(p, data), + rename: (from, to) => fsp.rename(from, to), + unlink: (p) => fsp.unlink(p).then(() => {}, () => {}), + validateMmdb: validateMmdbDefault, + // No-op by default — see the import-direction note at the top of this + // file. Real callers (server/location.ts's startup/timer hook and the + // admin refresh route in server/routes.ts) inject reloadGeoReaders. + reload: async () => {}, + }; +} + +let state: "idle" | "refreshing" = "idle"; +let lastResult: RefreshResult | null = null; + +export function getGeoipRefreshStatus(): { state: "idle" | "refreshing"; lastResult: RefreshResult | null } { + return { state, lastResult }; +} + +/** + * Downloads City + ASN into GEOIP_DIR, validating each before it replaces the + * canonical file. On ANY failure — network, extraction, or validation — the + * existing files and readers are left exactly as they were: the loop aborts + * before the rename that would swap in a new file, so a bad download never + * costs the site its previously-working GeoIP data. + */ +export async function refreshGeoipDatabases(overrides: Partial = {}): Promise { + const deps: RefreshDeps = { ...defaultDeps(), ...overrides }; + state = "refreshing"; + const at = deps.now.toISOString(); + let source: GeoipSource = "dbip"; + try { + const { key: maxmindKey } = await deps.getMaxmindKey(); + source = resolveGeoipSource(maxmindKey); + const urls = buildDownloadUrls(source, deps.now); + const files: Record = {}; + + for (const target of TARGETS) { + const primary = target.name === "City" ? urls.city : urls.asn; + const fallback = target.name === "City" ? urls.cityFallback : urls.asnFallback; + const requestUrl = source === "geolite2" && maxmindKey ? withLicenseKey(primary, maxmindKey) : primary; + + let raw: Buffer; + try { + raw = await deps.download(requestUrl); + } catch (err) { + const status = (err as { status?: number } | undefined)?.status; + if (source === "dbip" && fallback && status === 404) { + raw = await deps.download(fallback); + } else { + throw err; + } + } + + const mmdbBytes = source === "geolite2" + ? await deps.extractTarGz(raw, target.editionId) + : await deps.gunzip(raw); + + const tmpPath = path.join(deps.dir, `.tmp-${target.name}-${deps.now.getTime()}.mmdb`); + await deps.writeFile(tmpPath, mmdbBytes); + const valid = await deps.validateMmdb(tmpPath); + if (!valid) { + await deps.unlink(tmpPath); + throw new Error(`${target.name}.mmdb failed validation (size or format) after download`); + } + await deps.rename(tmpPath, path.join(deps.dir, `${target.name}.mmdb`)); + files[target.name] = { bytes: mmdbBytes.length }; + } + + const meta = { source, fetchedAt: at, files }; + await deps.writeFile(path.join(deps.dir, "geoip-meta.json"), JSON.stringify(meta, null, 2)); + await deps.reload(); + + const result: RefreshResult = { ok: true, source, at }; + lastResult = result; + state = "idle"; + return result; + } catch (err) { + const error = sanitizeErrorMessage(err instanceof Error ? err.message : String(err)); + console.error(`[geoip] refresh failed (source=${source}): ${error}`); + const result: RefreshResult = { ok: false, source, at, error }; + lastResult = result; + state = "idle"; + return result; + } +} + +/** + * Pure validation for the PUT .../maxmind-key body — trims, requires a + * non-empty value, and requires CREDENTIAL_ENCRYPTION_KEY to be set (the + * console-managed key is stored encrypted, same as the secrets feature; with + * no encryption key configured there is nowhere safe to put it). Kept + * separate from the route handler so this rejection logic is unit-testable + * without spinning up Express. + */ +export function validateMaxmindKeyInput(key: unknown): { ok: true; key: string } | { ok: false; error: string } { + const trimmed = typeof key === "string" ? key.trim() : ""; + if (!trimmed) return { ok: false, error: "key is required" }; + if (!isEncryptionConfigured()) { + return { + ok: false, + error: "Set CREDENTIAL_ENCRYPTION_KEY on the server to store a MaxMind license key from the admin console.", + }; + } + return { ok: true, key: trimmed }; +} + +/** Save-path helper for the admin route: encrypts and persists the console-managed key. */ +export async function saveMaxmindKey(key: string): Promise { + await storage.setConfig({ key: MAXMIND_KEY_CONFIG_KEY, value: encryptValue(key) }); +} + +export async function clearMaxmindKey(): Promise { + await storage.deleteConfig(MAXMIND_KEY_CONFIG_KEY); +} diff --git a/server/location.ts b/server/location.ts index 9dfa74a1..b6a0cb16 100644 --- a/server/location.ts +++ b/server/location.ts @@ -1,10 +1,11 @@ import { REGION_ELIGIBLE_TRUST, type LocationTrust } from "@shared/schema"; -import { open as maxmindOpen, type Reader, type CityResponse, type AsnResponse } from "maxmind"; +import { open as maxmindOpen, type Reader, type Response as MmdbResponse, type CityResponse, type AsnResponse } from "maxmind"; import { readFileSync } from "fs"; import path from "path"; import { storage } from "./storage"; import { getMarketplace } from "./marketplace"; import type { RegionCandidate } from "@shared/regions"; +import { refreshGeoipDatabases, DBIP_ATTRIBUTION, type GeoipSource } from "./geoip-refresh"; export type { RegionCandidate }; @@ -207,33 +208,85 @@ export interface DetectionDeps { asnClassLoaded: boolean; } +// Duplicated (not imported) from server/geoip-refresh.ts on purpose — see the +// import-direction note at the top of that file. geoip-refresh.ts never +// imports this module, so this module is the one that may import it back. const GEOIP_DIR = process.env.GEOIP_DB_DIR || path.join(process.cwd(), "geoip"); +const GEOIP_STALE_MS = 7 * 24 * 60 * 60 * 1000; +const GEOIP_REFRESH_INTERVAL_MS = 7 * 24 * 60 * 60 * 1000; + let cityReader: Reader | null = null; let asnReader: Reader | null = null; let torExits: Set = new Set(); let asnClassification: Record = {}; let asnClassificationLoaded = false; let warnedPrivateIp = false; -// Set when the GeoIP data source requires public credit (DB-IP Lite is -// CC-BY-4.0; scripts/geoip-refresh.sh writes geoip/ATTRIBUTION on that path -// and removes it on the MaxMind path). Exposed via /api/config → footer. +// Derived from geoip-meta.json's `source` field (written by refreshGeoipDatabases) +// whenever a DB is (re)loaded. DB-IP Lite is CC-BY-4.0 and needs public +// credit; GeoLite2 does not. Exposed via /api/config → footer. let geoipAttribution: string | null = null; export function getGeoipAttribution(): string | null { return geoipAttribution; } +async function tryOpen(names: string[], opener: (p: string) => Promise>): Promise | null> { + for (const name of names) { + try { return await opener(path.join(GEOIP_DIR, name)); } + catch { /* try the next name */ } + } + return null; +} + +function refreshAttributionFromMeta(): void { + try { + const meta = JSON.parse(readFileSync(path.join(GEOIP_DIR, "geoip-meta.json"), "utf8")) as { source?: GeoipSource }; + geoipAttribution = meta.source === "dbip" ? DBIP_ATTRIBUTION : null; + } catch { + geoipAttribution = null; // no meta yet (fresh checkout, or DBs never refreshed) = no attribution required + } +} + +/** + * Re-opens the mmdb readers from disk and swaps the module-level refs. Safe + * to call when the files are absent (readers become null; agents stay + * Unverified rather than the process crashing). Called once at startup and + * again after every successful refreshGeoipDatabases() call — it is the + * `reload` dependency startLocationServices() and the admin refresh route + * inject into refreshGeoipDatabases(). + */ +export async function reloadGeoReaders(): Promise { + // Canonical names first (City.mmdb/ASN.mmdb, written by the in-app + // refresher); GeoLite2-*.mmdb kept as a fallback for existing deployments + // that still have files under the old script's names. + const city = await tryOpen(["City.mmdb", "GeoLite2-City.mmdb"], maxmindOpen); + if (!city) console.log("[location] no City mmdb found — geolocation disabled (all agents Unverified)"); + cityReader = city; + + const asn = await tryOpen(["ASN.mmdb", "GeoLite2-ASN.mmdb"], maxmindOpen); + if (!asn) console.log("[location] no ASN mmdb found — ASN signals disabled"); + asnReader = asn; + + refreshAttributionFromMeta(); +} + export function startLocationServices(): void { void (async () => { - try { cityReader = await maxmindOpen(path.join(GEOIP_DIR, "GeoLite2-City.mmdb")); } - catch { console.log("[location] GeoLite2-City.mmdb not found — geolocation disabled (all agents Unverified)"); } - try { asnReader = await maxmindOpen(path.join(GEOIP_DIR, "GeoLite2-ASN.mmdb")); } - catch { console.log("[location] GeoLite2-ASN.mmdb not found — ASN signals disabled"); } + await reloadGeoReaders(); + let stale = true; + try { + const meta = JSON.parse(readFileSync(path.join(GEOIP_DIR, "geoip-meta.json"), "utf8")) as { fetchedAt?: string }; + const fetchedAt = meta.fetchedAt ? new Date(meta.fetchedAt).getTime() : NaN; + stale = !(Number.isFinite(fetchedAt) && Date.now() - fetchedAt < GEOIP_STALE_MS); + } catch { stale = true; } + const missing = !cityReader || !asnReader; + if (missing || stale) { + console.log(`[location] geoip refresh triggered at startup (${missing ? "database(s) missing" : "data older than 7 days"})`); + void refreshGeoipDatabases({ reload: reloadGeoReaders }); + } })(); - try { - const text = readFileSync(path.join(GEOIP_DIR, "ATTRIBUTION"), "utf8").trim(); - if (text) geoipAttribution = text; - } catch { /* no marker = no attribution required (MaxMind path or no DBs) */ } + setInterval(() => { void refreshGeoipDatabases({ reload: reloadGeoReaders }); }, GEOIP_REFRESH_INTERVAL_MS).unref(); + try { asnClassification = JSON.parse(readFileSync(path.join(process.cwd(), "server/data/asn-classification.json"), "utf8")); delete (asnClassification as Record)._comment; diff --git a/server/routes.ts b/server/routes.ts index 9125b57d..7fc79a77 100644 --- a/server/routes.ts +++ b/server/routes.ts @@ -16,7 +16,20 @@ import { parsePlatformSetup, sessionScopeForWorkflow, evaluateSessionRequirement import { validateRegisterPayload, cacheBrokerMintSecret, hasBrokerMintSecret } from "./broker-registry"; import { deriveApiKeyStatus } from "./api-key-status"; import { isStaleOfflineAgent } from "./agent-liveness"; -import { runAgentLocationCheck, LOCATION_RECHECK_HOURS, getGeoipAttribution } from "./location"; +import { runAgentLocationCheck, LOCATION_RECHECK_HOURS, getGeoipAttribution, reloadGeoReaders } from "./location"; +import { + refreshGeoipDatabases, + getGeoipRefreshStatus, + getMaxmindKey, + resolveGeoipSource, + saveMaxmindKey, + clearMaxmindKey, + validateMaxmindKeyInput, + DBIP_ATTRIBUTION, + GEOIP_DIR, +} from "./geoip-refresh"; +import { promises as fsp } from "fs"; +import path from "path"; import { hashPassword, verifyPassword, @@ -1064,6 +1077,86 @@ export async function registerRoutes( } }); + // ==================== GEOIP DATABASE REFRESH (ADMIN) ==================== + // See server/geoip-refresh.ts for the source-resolution and download logic. + // The MaxMind key request bodies below are never echoed back and never + // logged — the request-logging middleware in server/index.ts only ever + // captures the RESPONSE json (see isSensitiveResponsePath), never the + // request body, so there is nothing to add to server/sensitive-paths.ts + // here: these routes' responses never carry the key value either. + + app.get("/api/admin/geoip/status", requireAuth, requireAdmin, async (_req, res) => { + try { + const { key: maxmindKey, source: keySource } = await getMaxmindKey(); + const source = resolveGeoipSource(maxmindKey); + const dbFiles: Array<{ label: "City" | "ASN"; names: string[] }> = [ + { label: "City", names: ["City.mmdb", "GeoLite2-City.mmdb"] }, + { label: "ASN", names: ["ASN.mmdb", "GeoLite2-ASN.mmdb"] }, + ]; + const databases = await Promise.all(dbFiles.map(async ({ label, names }) => { + for (const name of names) { + try { + const stat = await fsp.stat(path.join(GEOIP_DIR, name)); + return { name: label, present: true, sizeBytes: stat.size, modifiedAt: stat.mtime.toISOString() }; + } catch { /* try the next name */ } + } + return { name: label, present: false, sizeBytes: null, modifiedAt: null }; + })); + const { state, lastResult } = getGeoipRefreshStatus(); + res.json({ + source, + dir: GEOIP_DIR, + state, + databases, + lastRefresh: lastResult, + attribution: source === "dbip" ? DBIP_ATTRIBUTION : null, + maxmindKey: { configured: !!maxmindKey, source: keySource }, + }); + } catch (error) { + console.error("Error fetching geoip status:", error); + res.status(500).json({ error: "Failed to fetch geoip status" }); + } + }); + + app.post("/api/admin/geoip/refresh", requireAuth, requireAdmin, async (_req, res) => { + try { + const { state } = getGeoipRefreshStatus(); + if (state === "refreshing") return res.status(409).json({ error: "A refresh is already in progress" }); + void refreshGeoipDatabases({ reload: reloadGeoReaders }); + res.status(202).json({ started: true }); + } catch (error) { + console.error("Error starting geoip refresh:", error); + res.status(500).json({ error: "Failed to start geoip refresh" }); + } + }); + + app.put("/api/admin/geoip/maxmind-key", requireAuth, requireAdmin, async (req, res) => { + try { + const validated = validateMaxmindKeyInput(req.body?.key); + if (!validated.ok) return res.status(400).json({ error: validated.error }); + await saveMaxmindKey(validated.key); + const { state } = getGeoipRefreshStatus(); + if (state === "refreshing") { + return res.status(202).json({ started: false, note: "A refresh is already in progress; the new key will be used on the next refresh." }); + } + void refreshGeoipDatabases({ reload: reloadGeoReaders }); + res.status(202).json({ started: true }); + } catch (error) { + console.error("Error saving MaxMind key:", error); + res.status(500).json({ error: "Failed to save MaxMind key" }); + } + }); + + app.delete("/api/admin/geoip/maxmind-key", requireAuth, requireAdmin, async (_req, res) => { + try { + await clearMaxmindKey(); + res.json({ deleted: true }); + } catch (error) { + console.error("Error clearing MaxMind key:", error); + res.status(500).json({ error: "Failed to clear MaxMind key" }); + } + }); + // ==================== USER PROFILE (SELF) ROUTES ==================== // Update own display name (username) diff --git a/server/storage.ts b/server/storage.ts index 923c767b..3a0145b5 100644 --- a/server/storage.ts +++ b/server/storage.ts @@ -2002,6 +2002,10 @@ export class DatabaseStorage { return inserted[0]; } + async deleteConfig(key: string): Promise { + await db.delete(systemConfig).where(eq(systemConfig.key, key)); + } + async createFundReturnRequest(request: InsertFundReturnRequest): Promise { const result = await db.insert(fundReturnRequests).values(request).returning(); return result[0]; diff --git a/tests/api.test.ts b/tests/api.test.ts index 958694df..0e26340a 100644 --- a/tests/api.test.ts +++ b/tests/api.test.ts @@ -4713,4 +4713,44 @@ describe('Vox API Tests', () => { expect(matching[0].id).toBe(brokerId); }); }); + + describe('GeoIP Admin API', () => { + it('should require admin for GET status', async () => { + const res = await fetch(`${BASE_URL}/api/admin/geoip/status`); + expect(res.status).toBe(401); + }); + + it('should require admin for POST refresh', async () => { + const res = await fetch(`${BASE_URL}/api/admin/geoip/refresh`, { method: 'POST' }); + expect(res.status).toBe(401); + }); + + it('should require admin for PUT maxmind-key', async () => { + const res = await fetch(`${BASE_URL}/api/admin/geoip/maxmind-key`, { + method: 'PUT', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ key: 'some-key' }), + }); + expect(res.status).toBe(401); + }); + + it('should require admin for DELETE maxmind-key', async () => { + const res = await fetch(`${BASE_URL}/api/admin/geoip/maxmind-key`, { method: 'DELETE' }); + expect(res.status).toBe(401); + }); + + it('admin GET status returns the expected shape without ever including a key value', async () => { + const res = await authFetch(adminSession, `${BASE_URL}/api/admin/geoip/status`); + expect(res.ok).toBe(true); + const body = await res.json(); + expect(['geolite2', 'dbip']).toContain(body.source); + expect(typeof body.dir).toBe('string'); + expect(['idle', 'refreshing']).toContain(body.state); + expect(Array.isArray(body.databases)).toBe(true); + expect(body.maxmindKey).toBeTruthy(); + expect(typeof body.maxmindKey.configured).toBe('boolean'); + // Never echoes a key value anywhere in the response. + expect(JSON.stringify(body)).not.toMatch(/"key"\s*:\s*"[^"]/); + }); + }); }); diff --git a/tests/geoip-live.test.ts b/tests/geoip-live.test.ts index 2399bf55..4c6aaf23 100644 --- a/tests/geoip-live.test.ts +++ b/tests/geoip-live.test.ts @@ -8,20 +8,32 @@ import { detectLocation, type DetectionDeps, type GeoLookup, type AsnLookup } fr * Live-database smoke test for the zero-trust detection pipeline. * * Runs ONLY when real mmdb files exist in GEOIP_DB_DIR (default ./geoip) — - * populate them with `scripts/geoip-refresh.sh` (works keyless via the DB-IP - * Lite fallback). CI and fresh checkouts have no geoip/ dir, so this file - * skips there; it exists to prove the ladder works against actual database - * content (field mapping, candidate derivation), which the unit tests' - * injected fixtures cannot. + * populate them by starting the server (server/location.ts refreshes on + * startup if missing/stale) or via the admin "Refresh" button/API + * (POST /api/admin/geoip/refresh, see server/geoip-refresh.ts; works keyless + * via the DB-IP Lite fallback). CI and fresh checkouts have no geoip/ dir, so + * this file skips there; it exists to prove the ladder works against actual + * database content (field mapping, candidate derivation), which the unit + * tests' injected fixtures cannot. * * Assertions are deliberately loose on city-level facts (GeoIP data shifts * between monthly editions) and firm on structural ones (schema fields * present, ladder outcomes, candidate shape). */ const GEOIP_DIR = process.env.GEOIP_DB_DIR || path.join(process.cwd(), "geoip"); -const haveDbs = - existsSync(path.join(GEOIP_DIR, "GeoLite2-City.mmdb")) && - existsSync(path.join(GEOIP_DIR, "GeoLite2-ASN.mmdb")); +// Canonical names (City.mmdb/ASN.mmdb, written by the in-app refresher) first, +// legacy GeoLite2-*.mmdb names kept for deployments that predate it — mirrors +// tryOpen() in server/location.ts. +function findDb(names: string[]): string | null { + for (const name of names) { + const p = path.join(GEOIP_DIR, name); + if (existsSync(p)) return p; + } + return null; +} +const cityDbPath = findDb(["City.mmdb", "GeoLite2-City.mmdb"]); +const asnDbPath = findDb(["ASN.mmdb", "GeoLite2-ASN.mmdb"]); +const haveDbs = !!cityDbPath && !!asnDbPath; const describeGeo = haveDbs ? describe : describe.skip; describeGeo("detection pipeline against live GeoIP databases", () => { @@ -30,8 +42,8 @@ describeGeo("detection pipeline against live GeoIP databases", () => { let deps: DetectionDeps; beforeAll(async () => { - cityReader = await open(path.join(GEOIP_DIR, "GeoLite2-City.mmdb")); - asnReader = await open(path.join(GEOIP_DIR, "GeoLite2-ASN.mmdb")); + cityReader = await open(cityDbPath!); + asnReader = await open(asnDbPath!); // Mirror liveDeps() from server/location.ts, minus Tor (network) and with // the checked-in ASN classification so hosting detection is exercised. const asnClass = (await import("../server/data/asn-classification.json")).default as Record< diff --git a/tests/geoip-refresh.test.ts b/tests/geoip-refresh.test.ts new file mode 100644 index 00000000..cd2d49f4 --- /dev/null +++ b/tests/geoip-refresh.test.ts @@ -0,0 +1,356 @@ +import { describe, it, expect, vi, afterEach } from "vitest"; +import { + resolveGeoipSource, + buildDownloadUrls, + refreshGeoipDatabases, + getGeoipRefreshStatus, + getMaxmindKey, + validateMaxmindKeyInput, + sanitizeErrorMessage, + downloadDefault, + DOWNLOAD_TIMEOUT_MS, + type RefreshDeps, +} from "../server/geoip-refresh"; +import { isEncryptionConfigured } from "../server/storage"; + +describe("resolveGeoipSource", () => { + it("picks geolite2 when a key is present", () => { + expect(resolveGeoipSource("abc123")).toBe("geolite2"); + }); + it("falls back to dbip when the key is null/undefined", () => { + expect(resolveGeoipSource(null)).toBe("dbip"); + expect(resolveGeoipSource(undefined)).toBe("dbip"); + }); + it("falls back to dbip when the key is an empty string", () => { + expect(resolveGeoipSource("")).toBe("dbip"); + }); +}); + +describe("buildDownloadUrls", () => { + it("geolite2: returns edition URLs with no license_key embedded", () => { + const urls = buildDownloadUrls("geolite2", new Date("2026-09-02T00:00:00Z")); + expect(urls.city).toContain("edition_id=GeoLite2-City"); + expect(urls.asn).toContain("edition_id=GeoLite2-ASN"); + expect(urls.city).not.toContain("license_key"); + expect(urls.asn).not.toContain("license_key"); + expect(urls.cityFallback).toBeUndefined(); + expect(urls.asnFallback).toBeUndefined(); + }); + + it("dbip: builds current-UTC-month URLs plus a previous-month fallback", () => { + const urls = buildDownloadUrls("dbip", new Date("2026-09-02T00:00:00Z")); + expect(urls.city).toBe("https://download.db-ip.com/free/dbip-city-lite-2026-09.mmdb.gz"); + expect(urls.cityFallback).toBe("https://download.db-ip.com/free/dbip-city-lite-2026-08.mmdb.gz"); + expect(urls.asn).toBe("https://download.db-ip.com/free/dbip-asn-lite-2026-09.mmdb.gz"); + expect(urls.asnFallback).toBe("https://download.db-ip.com/free/dbip-asn-lite-2026-08.mmdb.gz"); + }); + + it("dbip: rolls the previous month across a year boundary", () => { + const urls = buildDownloadUrls("dbip", new Date("2026-01-15T00:00:00Z")); + expect(urls.city).toBe("https://download.db-ip.com/free/dbip-city-lite-2026-01.mmdb.gz"); + expect(urls.cityFallback).toBe("https://download.db-ip.com/free/dbip-city-lite-2025-12.mmdb.gz"); + }); +}); + +describe("getMaxmindKey resolution order", () => { + it("console-managed key beats the env var", async () => { + // isEncryptionConfigured() gates whether we can even test the console + // path end-to-end (it needs CREDENTIAL_ENCRYPTION_KEY); when it's not + // configured in this environment we only assert the env fallback below. + if (!isEncryptionConfigured()) return; + const { storage, encryptValue } = await import("../server/storage"); + const original = process.env.MAXMIND_LICENSE_KEY; + process.env.MAXMIND_LICENSE_KEY = "env-key"; + try { + await storage.setConfig({ key: "maxmind_license_key", value: encryptValue("console-key") }); + const result = await getMaxmindKey(); + expect(result).toEqual({ key: "console-key", source: "console" }); + } finally { + await storage.deleteConfig("maxmind_license_key"); + if (original === undefined) delete process.env.MAXMIND_LICENSE_KEY; + else process.env.MAXMIND_LICENSE_KEY = original; + } + }); + + it("falls back to the env var when no console key is stored", async () => { + const { storage } = await import("../server/storage"); + await storage.deleteConfig("maxmind_license_key"); // ensure clean slate + const original = process.env.MAXMIND_LICENSE_KEY; + process.env.MAXMIND_LICENSE_KEY = "env-key"; + try { + const result = await getMaxmindKey(); + expect(result).toEqual({ key: "env-key", source: "env" }); + } finally { + if (original === undefined) delete process.env.MAXMIND_LICENSE_KEY; + else process.env.MAXMIND_LICENSE_KEY = original; + } + }); + + it("returns null/null when neither is set", async () => { + const { storage } = await import("../server/storage"); + await storage.deleteConfig("maxmind_license_key"); + const original = process.env.MAXMIND_LICENSE_KEY; + delete process.env.MAXMIND_LICENSE_KEY; + try { + const result = await getMaxmindKey(); + expect(result).toEqual({ key: null, source: null }); + } finally { + if (original !== undefined) process.env.MAXMIND_LICENSE_KEY = original; + } + }); +}); + +describe("validateMaxmindKeyInput", () => { + it("rejects an empty/whitespace-only key regardless of encryption config", () => { + expect(validateMaxmindKeyInput("")).toEqual({ ok: false, error: "key is required" }); + expect(validateMaxmindKeyInput(" ")).toEqual({ ok: false, error: "key is required" }); + expect(validateMaxmindKeyInput(undefined)).toEqual({ ok: false, error: "key is required" }); + expect(validateMaxmindKeyInput(42)).toEqual({ ok: false, error: "key is required" }); + }); + + it("rejects a non-empty key when CREDENTIAL_ENCRYPTION_KEY is not configured", () => { + const original = process.env.CREDENTIAL_ENCRYPTION_KEY; + delete process.env.CREDENTIAL_ENCRYPTION_KEY; + try { + const result = validateMaxmindKeyInput(" some-key "); + expect(result.ok).toBe(false); + if (!result.ok) expect(result.error).toContain("CREDENTIAL_ENCRYPTION_KEY"); + } finally { + if (original === undefined) delete process.env.CREDENTIAL_ENCRYPTION_KEY; + else process.env.CREDENTIAL_ENCRYPTION_KEY = original; + } + }); + + it("accepts and trims a non-empty key when encryption is configured", () => { + const original = process.env.CREDENTIAL_ENCRYPTION_KEY; + process.env.CREDENTIAL_ENCRYPTION_KEY = "a".repeat(64); // valid 32-byte hex + try { + expect(validateMaxmindKeyInput(" some-key ")).toEqual({ ok: true, key: "some-key" }); + } finally { + if (original === undefined) delete process.env.CREDENTIAL_ENCRYPTION_KEY; + else process.env.CREDENTIAL_ENCRYPTION_KEY = original; + } + }); +}); + +describe("sanitizeErrorMessage", () => { + it("redacts a license_key embedded in a real-shaped MaxMind fetch error", () => { + const msg = + "request to https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-City&license_key=SECRETVALUE123&suffix=tar.gz failed"; + const out = sanitizeErrorMessage(msg); + expect(out).not.toContain("SECRETVALUE123"); + expect(out).toContain("license_key=REDACTED"); + }); + + it("redacts case-insensitively and regardless of what follows (& or end of string)", () => { + expect(sanitizeErrorMessage("LICENSE_KEY=abc123&suffix=tar.gz")).toBe("license_key=REDACTED&suffix=tar.gz"); + expect(sanitizeErrorMessage("...license_key=abc123")).toBe("...license_key=REDACTED"); + }); + + it("leaves key-free messages untouched", () => { + expect(sanitizeErrorMessage("network down")).toBe("network down"); + }); +}); + +describe("downloadDefault (network timeout bound)", () => { + const originalFetch = global.fetch; + afterEach(() => { + global.fetch = originalFetch; + }); + + it("passes an AbortSignal to fetch so a hung request cannot wedge state forever", async () => { + const fetchMock = vi.fn(async (_url: string, init?: RequestInit) => { + expect(init?.signal).toBeInstanceOf(AbortSignal); + return new Response(Buffer.from("ok"), { status: 200 }); + }); + global.fetch = fetchMock as unknown as typeof fetch; + + const buf = await downloadDefault("https://example.com/x"); + expect(buf.toString()).toBe("ok"); + expect(fetchMock).toHaveBeenCalledTimes(1); + }); + + it("DOWNLOAD_TIMEOUT_MS is at least 120s (files run 50-90MB)", () => { + expect(DOWNLOAD_TIMEOUT_MS).toBeGreaterThanOrEqual(120_000); + }); + + it("a timed-out fetch flows through refreshGeoipDatabases's normal failure handling: state returns to idle and the error is recorded (sanitized)", async () => { + global.fetch = vi.fn(async () => { + throw new DOMException("The operation was aborted due to timeout", "TimeoutError"); + }) as unknown as typeof fetch; + + // Deliberately don't override `download` — this exercises the real + // downloadDefault (and therefore the real AbortSignal.timeout wiring) + // through the full refresh pipeline, not a mock. + const result = await refreshGeoipDatabases({ + getMaxmindKey: vi.fn(async () => ({ key: null, source: null as const })), + now: new Date("2026-09-02T00:00:00Z"), + }); + + expect(result.ok).toBe(false); + expect(result.error).toContain("aborted"); + expect(result.error).not.toContain("license_key"); + expect(getGeoipRefreshStatus().state).toBe("idle"); + }); +}); + +describe("refreshGeoipDatabases", () => { + function makeDeps(overrides: Partial = {}): RefreshDeps { + return { + dir: "/fake/geoip", + now: new Date("2026-09-02T00:00:00Z"), + getMaxmindKey: vi.fn(async () => ({ key: null, source: null as const })), // dbip path by default + download: vi.fn(async () => Buffer.from("raw-bytes")), + gunzip: vi.fn(async (data: Buffer) => Buffer.concat([Buffer.from("decompressed:"), data])), + extractTarGz: vi.fn(async () => Buffer.from("extracted-mmdb-bytes")), + writeFile: vi.fn(async () => {}), + rename: vi.fn(async () => {}), + unlink: vi.fn(async () => {}), + validateMmdb: vi.fn(async () => true), + reload: vi.fn(async () => {}), + ...overrides, + }; + } + + it("success path: downloads+decompresses both DBs, writes meta, renames into place, and reloads", async () => { + const deps = makeDeps(); + const result = await refreshGeoipDatabases(deps); + + expect(result.ok).toBe(true); + expect(result.source).toBe("dbip"); + expect(deps.download).toHaveBeenCalledTimes(2); // city + asn + expect(deps.gunzip).toHaveBeenCalledTimes(2); + expect(deps.extractTarGz).not.toHaveBeenCalled(); + expect(deps.rename).toHaveBeenCalledTimes(2); // City.mmdb, ASN.mmdb + expect(deps.rename).toHaveBeenCalledWith(expect.any(String), "/fake/geoip/City.mmdb"); + expect(deps.rename).toHaveBeenCalledWith(expect.any(String), "/fake/geoip/ASN.mmdb"); + expect(deps.writeFile).toHaveBeenCalledTimes(3); // City tmp, ASN tmp, geoip-meta.json + expect(deps.reload).toHaveBeenCalledTimes(1); + + const metaCall = (deps.writeFile as ReturnType).mock.calls.find( + ([target]) => target === "/fake/geoip/geoip-meta.json", + ); + expect(metaCall).toBeTruthy(); + const meta = JSON.parse(metaCall![1].toString()); + expect(meta.source).toBe("dbip"); + expect(meta.fetchedAt).toBe("2026-09-02T00:00:00.000Z"); + expect(meta.files.City).toBeTruthy(); + expect(meta.files.ASN).toBeTruthy(); + + const status = getGeoipRefreshStatus(); + expect(status.state).toBe("idle"); + expect(status.lastResult?.ok).toBe(true); + expect(status.lastResult?.source).toBe("dbip"); + }); + + it("uses extractTarGz (not gunzip) for the geolite2 source and never leaks the license key", async () => { + const deps = makeDeps({ getMaxmindKey: vi.fn(async () => ({ key: "super-secret-key", source: "env" as const })) }); + const result = await refreshGeoipDatabases(deps); + + expect(result.ok).toBe(true); + expect(result.source).toBe("geolite2"); + expect(deps.extractTarGz).toHaveBeenCalledTimes(2); + expect(deps.gunzip).not.toHaveBeenCalled(); + + // The recorded result/status must never contain the raw key value, even + // though it was necessarily composed into the (mocked) download URL. + expect(JSON.stringify(result)).not.toContain("super-secret-key"); + const status = getGeoipRefreshStatus(); + expect(JSON.stringify(status)).not.toContain("super-secret-key"); + }); + + it("dbip: falls back to the previous month on a 404 for the current month", async () => { + const urls = buildDownloadUrls("dbip", new Date("2026-09-02T00:00:00Z")); + const download = vi.fn(async (url: string) => { + if (url === urls.city) { + const err = new Error("HTTP 404") as Error & { status?: number }; + err.status = 404; + throw err; + } + return Buffer.from("raw-bytes"); + }); + const deps = makeDeps({ download }); + const result = await refreshGeoipDatabases(deps); + + expect(result.ok).toBe(true); + expect(download).toHaveBeenCalledWith(urls.cityFallback); + expect(download).toHaveBeenCalledWith(urls.asn); + }); + + it("validation failure: keeps old files (no rename of the canonical name) and records the error", async () => { + const rename = vi.fn(async () => {}); + const deps = makeDeps({ validateMmdb: vi.fn(async () => false), rename }); + const result = await refreshGeoipDatabases(deps); + + expect(result.ok).toBe(false); + expect(result.error).toBeTruthy(); + expect(rename).not.toHaveBeenCalled(); + expect(deps.reload).not.toHaveBeenCalled(); + + const status = getGeoipRefreshStatus(); + expect(status.state).toBe("idle"); + expect(status.lastResult?.ok).toBe(false); + expect(status.lastResult?.error).toBeTruthy(); + }); + + it("download failure: records the error without throwing and without calling reload", async () => { + const deps = makeDeps({ + download: vi.fn(async () => { throw new Error("network down"); }), + }); + const result = await refreshGeoipDatabases(deps); + + expect(result.ok).toBe(false); + expect(result.error).toContain("network down"); + expect(deps.reload).not.toHaveBeenCalled(); + }); + + it("a key-bearing download error is redacted in both lastResult.error and the logged line — GET /api/admin/geoip/status echoes lastResult.error verbatim, and the request-logging middleware writes that response to the server log, so this is the sole guard against a leak", async () => { + const consoleErrorSpy = vi.spyOn(console, "error").mockImplementation(() => {}); + try { + const keyBearingMessage = + "request to https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-City&license_key=SECRETVALUE123&suffix=tar.gz failed"; + const deps = makeDeps({ + getMaxmindKey: vi.fn(async () => ({ key: "SECRETVALUE123", source: "env" as const })), + download: vi.fn(async () => { throw new Error(keyBearingMessage); }), + }); + const result = await refreshGeoipDatabases(deps); + + expect(result.ok).toBe(false); + expect(result.error).toBeTruthy(); + expect(result.error).not.toContain("SECRETVALUE123"); + expect(result.error).toContain("license_key=REDACTED"); + + // What GET /api/admin/geoip/status would actually serialize and what the + // request-logging middleware would write to the log — assert on the + // same JSON.stringify a real response body goes through. + expect(JSON.stringify(getGeoipRefreshStatus())).not.toContain("SECRETVALUE123"); + + // The one console.error line the module emits per failed refresh. + expect(consoleErrorSpy).toHaveBeenCalledTimes(1); + const loggedLine = consoleErrorSpy.mock.calls[0][0] as string; + expect(loggedLine).not.toContain("SECRETVALUE123"); + expect(loggedLine).toContain("license_key=REDACTED"); + } finally { + consoleErrorSpy.mockRestore(); + } + }); + + it("a failing geolite2 download after a healthy dbip state leaves existing files/readers untouched", async () => { + // Simulates: DB-IP files already on disk and serving, operator saves a bad + // MaxMind key, refresh is triggered, the GeoLite2 download fails. The + // atomic rename-only-after-validation design means this can never disturb + // the files already in place — assert the failure path never renames. + const rename = vi.fn(async () => {}); + const deps = makeDeps({ + getMaxmindKey: vi.fn(async () => ({ key: "bad-key", source: "env" as const })), + download: vi.fn(async () => { throw new Error("HTTP 401 Unauthorized"); }), + rename, + }); + const result = await refreshGeoipDatabases(deps); + + expect(result.ok).toBe(false); + expect(result.source).toBe("geolite2"); + expect(rename).not.toHaveBeenCalled(); + expect(deps.reload).not.toHaveBeenCalled(); + }); +});