diff --git a/.github/workflows/pull_requests.yaml b/.github/workflows/pull_requests.yaml index 7548a6c50..b1c98f23d 100644 --- a/.github/workflows/pull_requests.yaml +++ b/.github/workflows/pull_requests.yaml @@ -24,6 +24,10 @@ jobs: GH_API_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | node _build_scripts/update-config-versions.js + # Structural checks on the /e/ error-message redirector. No build needed, + # so it runs first and fails fast. + - name: Validate the error-message redirects + run: node _build_scripts/validate-redirects.js - name: Build a dev version run: yarn build-dev - name: Validate links from the build.dev folder diff --git a/_build_scripts/validate-links-pr.js b/_build_scripts/validate-links-pr.js index 5969fa64d..da4838303 100644 --- a/_build_scripts/validate-links-pr.js +++ b/_build_scripts/validate-links-pr.js @@ -50,6 +50,10 @@ const runPRValidationFromBuildDev = async () => { `/cloud`, `/engram`, `/contributor-guide`, + // Out of the site navigation, so nothing links down into it from the + // sections above and the crawler would otherwise never reach it. + `/errors`, + `/improve-your-cluster`, ] const success = await validator.validateLinks(paths); diff --git a/_build_scripts/validate-redirects.js b/_build_scripts/validate-redirects.js new file mode 100644 index 000000000..b105cefdf --- /dev/null +++ b/_build_scripts/validate-redirects.js @@ -0,0 +1,245 @@ +/** + * Structural checks on the /e/ error-message redirector in netlify.toml. + * + * These invariants are load-bearing and none of them is enforced by anything + * else: Netlify does not validate the file, the Docusaurus build never reads + * it, and the link crawler only sees pages. They are also each one careless + * edit away from breaking, which is why this exists rather than a comment. + * + * Reads netlify.toml and the MDX under docs/. Needs no build and no network. + * + * node _build_scripts/validate-redirects.js + */ + +const fs = require('fs') +const path = require('path') + +const ROOT = path.resolve(__dirname, '..') +const TOML = path.join(ROOT, 'netlify.toml') +const BLOCK_MARKER = 'ERROR-MESSAGE REDIRECTOR' + +const failures = [] +const fail = (msg) => failures.push(msg) + +const toml = fs.readFileSync(TOML, 'utf8') +const markerAt = toml.indexOf(BLOCK_MARKER) +if (markerAt === -1) { + console.error(`Could not find the "${BLOCK_MARKER}" block in netlify.toml.`) + process.exit(1) +} + +// Comments are stripped first so that prose in the block (which discusses these +// very rules, and quotes them) can never be read as a rule or counted as one. +const block = toml + .slice(markerAt) + .split('\n') + .filter((line) => !/^[ \t]*#/.test(line)) + .join('\n') + +/** + * Parse [[redirects]] tables into plain objects. + * + * Deliberately NOT one regex over from/to/status in a fixed order with fixed + * spacing. TOML does not care about key order or whitespace, and this file is + * hand-edited, so a checker that only recognises today's formatting would go + * quietly blind on exactly the edit it exists to catch. Reads whatever keys are + * present, in any order, single- or double-quoted. + */ +const parseRedirectTables = (text) => + text + .split(/\[\[redirects\]\]/) + .slice(1) + .map((chunk) => { + // A table ends at the next TOML header of any kind. + const body = chunk.split(/\n[ \t]*\[/)[0] + const KV = /^[ \t]*([A-Za-z_][A-Za-z0-9_-]*)[ \t]*=[ \t]*(?:"([^"]*)"|'([^']*)'|([^\s#]+))/gm + const entry = {} + for (const m of body.matchAll(KV)) { + entry[m[1]] = m[2] !== undefined ? m[2] : m[3] !== undefined ? m[3] : m[4] + } + return entry + }) + +const tables = parseRedirectTables(block) +const rules = tables.filter((t) => typeof t.from === 'string' && t.from.startsWith('/e/')) + +// --------------------------------------------------------------------------- +// 0. THE PARSE ITSELF MUST BE COMPLETE. +// +// Every check below reasons over `rules`. If the parser silently drops a rule, +// each of them still passes, and the script reports success over a set it never +// examined. That is a worse outcome than having no checker at all, and it is +// the same failure shape as a link checker that validates zero files. So count +// the rules independently, as loosely as possible, and refuse to continue if +// the two numbers disagree. +// --------------------------------------------------------------------------- +const declared = (block.match(/^[ \t]*from[ \t]*=[ \t]*["']?\/e\//gm) || []).length +if (declared !== rules.length) { + console.error( + `\nnetlify.toml /e/ redirector: PARSE GAP.\n\n` + + ` ${declared} rule(s) declare a /e/ source, but only ${rules.length} parsed.\n` + + ` Every check in this script reasons over the parsed set, so it cannot be\n` + + ` trusted until they agree. Fix the parser in ${path.basename(__filename)}\n` + + ` rather than reformatting netlify.toml to suit it.\n` + ) + process.exit(1) +} + +if (rules.length === 0) fail('No /e/ redirect rules found.') + +// Each rule must actually be a rule. +for (const r of rules) { + if (!r.to) fail(`"${r.from}" has no "to" target.`) + if (!r.status) fail(`"${r.from}" has no "status".`) +} + +const splats = rules.filter((r) => r.from.includes('*')) +const specific = rules.filter((r) => !r.from.includes('*')) +const last = rules[rules.length - 1] + +// 1. Exactly one catch-all, and it is the last /e/ rule. A rule appended after +// it would be unreachable, because the catch-all matches everything first. +if (splats.length !== 1) { + fail(`Expected exactly one splat rule in the /e/ block, found ${splats.length}: ${splats.map((r) => r.from).join(', ')}`) +} else if (splats[0].from !== '/e/*') { + fail(`The only splat should be "/e/*", found "${splats[0].from}".`) +} else if (last.from !== '/e/*') { + fail(`The catch-all must be the LAST /e/ rule, but "${last.from}" follows it. Rules after the catch-all can never match.`) +} + +// 2. Status codes. A specific id maps to one meaning forever, so 301 and an +// indefinite browser cache are correct. The catch-all is the opposite: it +// fires for ids whose entry is not written yet, so its destination changes +// as soon as one is, and a cached 301 could never be corrected. +for (const r of specific) { + if (r.status !== '301') fail(`"${r.from}" should be status 301, found ${r.status}.`) +} +if (splats.length === 1 && splats[0].status !== '302') { + fail(`The "/e/*" catch-all should be status 302, found ${splats[0].status}. See the comment above it in netlify.toml.`) +} + +// 3. Lowercase ids. Netlify matches paths case-sensitively, and messages print +// the id in mixed case for readability ("Dep004"), so a mixed-case rule here +// silently never fires. +for (const r of rules) { + if (r.from !== r.from.toLowerCase()) fail(`Redirect sources must be lowercase: "${r.from}".`) +} + +// 4. No duplicate sources: a second rule for the same id is dead, because the +// first one always wins, and it reads as if it were in effect. +const seen = new Set() +for (const r of rules) { + if (seen.has(r.from)) fail(`Duplicate rule for "${r.from}" (the later one can never match).`) + seen.add(r.from) +} + +// 5. No destination may carry a query string of its own. +// +// Netlify forwards an incoming query to the destination only when the +// destination has none. Measured with `netlify dev` and confirmed against +// production: +// +// to = "/errors/x#a" + ?clusterid=U -> /errors/x?clusterid=U#a +// to = "/errors/x?src=id#a" + ?clusterid=U -> /errors/x?src=id#a +// +// So a `?` here silently throws away the query the request arrived with, +// including the `?clusterid=` Weaviate puts on these links, with no +// build error and no broken link to notice. The message id is added by +// netlify/edge-functions/error-link-src.ts instead, which is why every +// destination below can stay plain. +for (const r of rules) { + if (r.to && r.to.includes('?')) { + fail( + `"${r.from}" has a query string in its destination ("${r.to}"). ` + + `Netlify drops the reader's own query when the destination has one, ` + + `so this would discard ?clusterid=. Let error-link-src.ts add the id instead.` + ) + } +} + +// 6. At most one fragment. (Check 5 already covers the other half of this: +// "/x#a?b" is not a URL with a query, the "?b" is part of the fragment and +// never reaches the server, and it trips the "?" test above.) +for (const r of rules) { + if (!r.to) continue + const hashes = (r.to.match(/#/g) || []).length + if (hashes > 1) fail(`"${r.from}" has ${hashes} "#" in its destination ("${r.to}"). A URL has at most one fragment.`) +} + +// 7. The edge function must be present and wired to /e/*. +// +// It is the only thing that puts `src=` on the destination URL, since no +// `to` value here may carry a query string (check 5). Deleting or unwiring +// the file leaves the redirects working and the parameter silently gone, and +// nothing else would notice. +const EDGE_FN = path.join(ROOT, 'netlify', 'edge-functions', 'error-link-src.ts') +if (!fs.existsSync(EDGE_FN)) { + fail( + 'netlify/edge-functions/error-link-src.ts is missing. It is what puts the ' + + 'message id on the destination URL as ?src=; without it the /e/ links ' + + 'still resolve but the parameter is gone.' + ) +} else { + // Comments are stripped first. The file's header quotes these settings + // verbatim, so a check run over the raw source passes on the documentation of + // a setting that has been deleted -- which is how this check failed its own + // negative test the first time it was written. + const fn = fs + .readFileSync(EDGE_FN, 'utf8') + .replace(/\/\*[\s\S]*?\*\//g, '') + .replace(/^[ \t]*\/\/.*$/gm, '') + if (!/path:\s*["']\/e\/\*["']/.test(fn)) { + fail('error-link-src.ts no longer declares path: "/e/*", so it will not run for these rules.') + } + if (!/onError:\s*["']bypass["']/.test(fn)) { + fail( + 'error-link-src.ts no longer sets onError: "bypass". Without it a thrown ' + + 'error turns every /e/ link into a 500 instead of falling through to the ' + + 'redirect rules below.' + ) + } +} + +// 8. Destinations inside /errors must resolve to a real page AND a real anchor. +// Nothing downstream catches a bad fragment: the build only warns on broken +// anchors it finds in page links, and it never sees this file at all. +const docsPath = (route) => { + const rel = route.replace(/^\//, '') + for (const candidate of [`docs/${rel}.mdx`, `docs/${rel}.md`, `docs/${rel}/index.mdx`, `docs/${rel}/index.md`]) { + const full = path.join(ROOT, candidate) + if (fs.existsSync(full)) return full + } + return null +} + +let checkedDestinations = 0 +for (const r of rules) { + if (!r.to || !r.to.startsWith('/errors')) continue + const [route, fragment] = r.to.split('#') + const file = docsPath(route) + if (!file) { + fail(`"${r.from}" points at "${route}", which has no page under docs/.`) + continue + } + checkedDestinations++ + if (!fragment) continue + const body = fs.readFileSync(file, 'utf8') + // Entries use explicit {#anchor} overrides, so several ids can share one + // heading without the anchor being tied to the heading's wording. + if (!body.includes(`{#${fragment}}`)) { + fail(`"${r.from}" points at "#${fragment}", which is not an explicit {#...} anchor in ${path.relative(ROOT, file)}.`) + } +} + +if (failures.length > 0) { + console.error(`\nnetlify.toml /e/ redirector: ${failures.length} problem(s)\n`) + for (const f of failures) console.error(` - ${f}`) + console.error('') + process.exit(1) +} + +console.log( + `netlify.toml /e/ redirector OK: ${rules.length} rules parsed (all ${declared} declared), ` + + `${specific.length} specific (301), 1 catch-all (302, last), ` + + `${checkedDestinations} destinations in /errors verified to a real page and anchor.` +) diff --git a/docs/errors/cluster-resources.mdx b/docs/errors/cluster-resources.mdx new file mode 100644 index 000000000..484387931 --- /dev/null +++ b/docs/errors/cluster-resources.mdx @@ -0,0 +1,86 @@ +--- +title: Cluster resource messages +description: "Weaviate log messages that mean a node has run out of a resource, such as memory mappings, with the cause and the fix for each one." +image: og/default.jpg +hide_table_of_contents: true +side_panel: improve-cluster +--- + +import ClusterIdNotice from '@site/src/components/ClusterIdNotice'; + + + +Messages on this page mean that a Weaviate node asked the operating system for a resource and was refused. The data is intact, but the node cannot open the files it needs until the limit is raised or the load is reduced. + +If your message is not here, the [message index](/errors) lists the other groups. + +## Not enough memory mappings {#not-enough-memory-mappings} + +Message ids: `core-mem001` + +**What you see** + +An error in the node's logs, ending in `not enough memory mappings`: + +```text +memory pressure: cannot init shard: not enough memory mappings +memory pressure: cannot load shard: not enough memory mappings +``` + +Which entry carries it depends on the operation that hit the limit. These are three separate log entries, and you may see any of them: + +| Log message | `action` field | Where to find `not enough memory mappings` | +| --- | --- | --- | +| `failed to load shard: memory pressure: cannot load shard: not enough memory mappings` | `load_shard` | In the message | +| `loading shard "MyTenant" failed` | `tenant_activation_lazy_load_shard` | Either appended to that message or in a separate `error` field, depending on your version | +| `failed to reload local index` | (none) | In a separate `error` field | + +Weaviate logs JSON by default, so for the second and third entries the cause can sit in a sibling `error` field rather than in the message. Search the `error` field as well as the message, and search for `not enough memory mappings` rather than for a whole line: that phrase is the one part that appears somewhere in the entry on every version. + +Collections or tenants whose shards did not load are unavailable for reads and writes, and the failure often shows up alongside replication errors such as `broadcast: cannot reach enough replicas`. + +**Why it happens** + +Every shard memory-maps several files, and the Linux kernel caps how many mappings one process may hold. That cap is `vm.max_map_count`. Weaviate reads it at startup and budgets 70% of it, leaving room for other processes on the host, then refuses to open a shard that would take it past that budget. Refusing early is deliberate: exhausting the kernel limit outright would fail unpredictably somewhere else in the process. + +The usual trigger is scale rather than a bug. Distribution defaults for `vm.max_map_count` were chosen for ordinary workloads, and a node holding thousands of active tenants or collections passes them. + +**How to fix it** + +1. Read the current limit on the node that logged the error: + + ```bash + sysctl vm.max_map_count + ``` + +2. Raise it. Three to four times the current value is a reasonable first step, and the value is a count, not an amount of memory, so raising it costs nothing on its own: + + ```bash + sysctl -w vm.max_map_count=8388608 + ``` + +3. Make it survive a reboot: + + ```bash + echo "vm.max_map_count=8388608" >> /etc/sysctl.conf + ``` + +4. Restart the affected node, or the affected pod on Kubernetes, so Weaviate reads the new limit. + +On Kubernetes the setting belongs to the host kernel, not to the container, so apply it on each node through your node configuration, a `DaemonSet`, or a privileged init container. Setting it inside an unprivileged container has no effect. On Weaviate Cloud the host configuration is not yours to change, so open a support ticket instead. + +If the limit is already high and the error keeps coming back, the node is holding more shards than it was sized for. Deactivate or offload tenants that are not in use, or add nodes and rebalance. + +**Learn more** + +- [Known issues: memory pressure, shard init failure](/weaviate/release-notes/known-issues#memory-pressure-shard-init-failure) +- [Resource planning](/weaviate/concepts/resources) +- [Tenant states and offloading](/weaviate/manage-collections/tenant-states) +- [Deployment troubleshooting](/deploy/faqs/troubleshooting) +- [Improve your cluster](/improve-your-cluster) + +## Questions and feedback + +import DocsFeedback from '/_includes/docs-feedback.mdx'; + + diff --git a/docs/errors/collection-configuration.mdx b/docs/errors/collection-configuration.mdx new file mode 100644 index 000000000..2a7fd663b --- /dev/null +++ b/docs/errors/collection-configuration.mdx @@ -0,0 +1,242 @@ +--- +title: Collection configuration messages +description: "Deprecation warnings raised when a collection describes its vectors, vector index, or multi-vector settings in a retired way, with the current form for each." +image: og/default.jpg +hide_table_of_contents: true +side_panel: improve-cluster +--- + +import ClusterIdNotice from '@site/src/components/ClusterIdNotice'; + + + +Messages on this page are deprecation warnings from the Weaviate Python client. Nothing has failed: the collection was created or updated as you asked. They warn that the way the request described its vectors is on its way out and will stop working in a future release, so the fix is always a rewrite of the configuration, never a change to your data. + +If your message is not here, the [message index](/errors) lists the other groups. + +## Deprecated vector configuration arguments {#deprecated-vector-configuration} + +Message ids: `py-dep017`, `py-dep023`, `py-dep024`, `py-dep025` + +**What you see** + +One or more of these warnings when you create or update a collection: + +```text +Dep024: You are using the `vectorizer_config` argument in `collection.config.create()`, which is deprecated. + Use the `vector_config` argument instead. + +Dep025: You are using the `vector_index_config` argument in `collection.config.create()`, which is deprecated. + Use the `vector_config` argument instead defining `vector_index_config` as a sub-argument. + +Dep017: You are using the `vector_index_config` argument in the `collection.config.update()` method, which is deprecated. + Use the `vector_config` argument instead. + +Dep023: You are using the `vectorizer_config` argument in the `collection.config.update()` method with a collection with named vectors, which is deprecated. + Use the `vector_config` argument instead. +``` + +**Why it happens** + +A collection used to describe its embeddings as two independent settings: one for the model that produces the vector, and one for the index that stores it. That shape cannot express a collection with several differently configured vectors, which Weaviate supports. Both settings now live inside a single vector definition, and a collection holds one or more of them, each with a name. + +The old arguments still work, and a collection created with them keeps the older single-vector shape. + +**How to fix it** + +Move the model and the index settings into one vector definition when you create the collection: + +```python +from weaviate.classes.config import Configure + +# Deprecated +client.collections.create( + "Article", + vectorizer_config=Configure.Vectorizer.text2vec_weaviate(), + vector_index_config=Configure.VectorIndex.hnsw(), +) + +# Current +client.collections.create( + "Article", + vector_config=Configure.Vectors.text2vec_weaviate( + vector_index_config=Configure.VectorIndex.hnsw(), + ), +) +``` + +Updates name the vector they change. A collection created with the current argument, and no explicit vector name, has a single vector called `default`: + +```python +from weaviate.classes.config import Reconfigure + +# Deprecated +client.collections.get("Article").config.update( + vector_index_config=Reconfigure.VectorIndex.hnsw(ef=128), +) + +# Current +client.collections.get("Article").config.update( + vector_config=Reconfigure.Vectors.update( + name="default", + vector_index_config=Reconfigure.VectorIndex.hnsw(ef=128), + ), +) +``` + +:::caution An existing collection is not converted in place + +Only a collection that was created with the current argument has a named vector for the current update form to address. A collection created with the deprecated arguments keeps the older shape, so updating it goes on raising the warning until the collection is recreated. + +The two forms are not interchangeable, and using the wrong one for a collection's shape fails rather than warns. If an update raised one of these, it tells you which shape you have: + +| What you got back | What it means | What to do | +| --- | --- | --- | +| `KeyError: 'vectorConfig'` | The collection has no named vectors, because it was created with the deprecated arguments | Keep using the deprecated argument for this collection until you recreate it | +| `KeyError: 'vectorIndexConfig'` | The opposite: the collection has named vectors, so the deprecated argument has nothing to address | Switch this call to the current form | +| `WeaviateInvalidInputError: Invalid input provided: Vector config with name default does not exist in the existing vector config.` | The collection has named vectors, but none by that name | Use the name the collection gave its vector, which is only `default` if none was set | + +Change the code that creates collections first. That is where the deprecation actually bites, because it is the form that will stop working. + +::: + +Rewriting a collection definition does not re-embed anything and does not touch stored vectors. It changes how the collection is described, so the same request keeps working once the old arguments are removed. + +**Learn more** + +- [Configure vectors for a collection](/weaviate/manage-collections/vector-config) +- [Collection configuration reference](/weaviate/config-refs/collections) +- [Python client: vectorizer API changes](/weaviate/client-libraries/python#vectorizer-api-changes-v4160) + +## Deprecated named vector syntax {#deprecated-named-vector-syntax} + +Message ids: `py-dep026` + +**What you see** + +A warning when you add a vector to an existing collection: + +```text +Dep026: You are using the named vector syntax for vector title_vector, e.g. `Configure.NamedVectors` in `collection.config.add_vector()`, which is deprecated. + Use `Configure.Vectors` or `Configure.MultiVectors` instead.` +``` + +The vector name in the message is your own, so it differs from the one shown here. + +**Why it happens** + +When named vectors were introduced they had a separate set of builders, kept apart from the single-vector ones. Every vector is now a named vector, so the two sets were merged into one, split instead by whether the vector is a single vector or a multi-vector. The old builders are aliases that still work. + +**How to fix it** + +Use the current builder for the kind of vector you are adding. The arguments are unchanged: + +```python +from weaviate.classes.config import Configure + +collection = client.collections.get("Article") + +# Deprecated +collection.config.add_vector( + vector_config=Configure.NamedVectors.text2vec_weaviate( + name="title_vector", + source_properties=["title"], + ), +) + +# Current +collection.config.add_vector( + vector_config=Configure.Vectors.text2vec_weaviate( + name="title_vector", + source_properties=["title"], + ), +) +``` + +For a multi-vector embedding, such as ColBERT or ColPali, use the multi-vector builders instead. If your warning names the `encoding` argument rather than the named vector syntax, it is the other message that ships under this same id: see [deprecated multi-vector index settings](#deprecated-multi-vector-settings). + +**Learn more** + +- [Configure vectors for a collection](/weaviate/manage-collections/vector-config) +- [Add a vector to an existing collection](/weaviate/manage-collections/vector-config#add-new-named-vectors) + +## Deprecated multi-vector index settings {#deprecated-multi-vector-settings} + +Message ids: `py-dep026`, `py-dep027` + +**What you see** + +One or both of these warnings when you create a collection with a multi-vector embedding: + +```text +Dep027: You are using the `multi_vector` argument in `Configure.VectorIndex.hnsw()`, which is deprecated. + Use the `multi_vector` argument inside `Configure.MultiVectors.module()` instead. + +Dep026: You are using the `encoding` argument in `Configure.VectorIndex.MultiVectors.multi_vector()`, which is deprecated. + Use the `encoding` argument inside `Configure.MultiVectors.module()` instead. +``` + +The second one is a different message that ships under the same id as [deprecated named vector syntax](#deprecated-named-vector-syntax). It has the same cause and the same fix as the one above it, so both are answered here. You get both warnings at once if you set an encoding inside the index configuration, because that nests one deprecated argument inside the other. + +The second message is quoted exactly as it prints, and the path inside it is wrong. It names `Configure.VectorIndex.MultiVectors`, plural, which does not exist. The real one is `Configure.VectorIndex.MultiVector`, singular. The plural spelling is easy to reach for because `Configure.MultiVectors` does exist at the top level, so copying the path out of the warning fails. The working form is in the snippets below. + +**Why it happens** + +Whether a vector is a multi-vector is a property of the vector, not of the index that stores it, and it is already implied by the multi-vector builder you chose. Declaring it a second time on the index left two places that could disagree, so the setting moved onto the vector definition. + +**How to fix it** + +Move the setting out of the index configuration and onto the vector definition. In the message, the second part of the name stands for whichever multi-vector builder you are using, such as a self-provided vector or a specific model integration: + +```python +from weaviate.classes.config import Configure + +# Deprecated +client.collections.create( + "Document", + vector_config=Configure.MultiVectors.self_provided( + name="page_vector", + vector_index_config=Configure.VectorIndex.hnsw( + multi_vector=Configure.VectorIndex.MultiVector.multi_vector(), + ), + ), +) + +# Current +client.collections.create( + "Document", + vector_config=Configure.MultiVectors.self_provided( + name="page_vector", + multi_vector_config=Configure.VectorIndex.MultiVector.multi_vector(), + vector_index_config=Configure.VectorIndex.hnsw(), + ), +) +``` + +An encoding such as MUVERA moves the same way, onto the vector definition: + +```python +from weaviate.classes.config import Configure + +# Current +client.collections.create( + "Document", + vector_config=Configure.MultiVectors.self_provided( + name="page_vector", + encoding=Configure.VectorIndex.MultiVector.Encoding.muvera(), + vector_index_config=Configure.VectorIndex.hnsw(), + ), +) +``` + +**Learn more** + +- [Multi-vector embeddings](/weaviate/manage-collections/vector-config#define-multi-vector-embeddings-eg-colbert-colpali) +- [Multi-vector embeddings tutorial](/weaviate/tutorials/multi-vector-embeddings) +- [Multi-vector compression](/weaviate/configuration/compression/multi-vectors) + +## Questions and feedback + +import DocsFeedback from '/_includes/docs-feedback.mdx'; + + diff --git a/docs/errors/index.mdx b/docs/errors/index.mdx new file mode 100644 index 000000000..cd32eccc3 --- /dev/null +++ b/docs/errors/index.mdx @@ -0,0 +1,76 @@ +--- +title: Error and warning messages +description: "Look up a Weaviate error, warning, or log message by its text or its message id, and get the cause and the fix on one page." +image: og/default.jpg +hide_table_of_contents: true +side_panel: improve-cluster +--- + +import ClusterIdNotice from '@site/src/components/ClusterIdNotice'; + + + +The Weaviate Python client prints a short id at the front of many of its warnings, such as `Dep024`. Weaviate Database does not print the id as a prefix. Instead, a log entry that has a documented cause is gaining a `docs_url` field that carries the id inside a link. Weaviate logs JSON by default, so it arrives as a key on the entry: + +```json +"docs_url": "https://docs.weaviate.io/e/core-mem001" +``` + +With `LOG_FORMAT=text` the same field prints as `docs_url=https://docs.weaviate.io/e/core-mem001`. + +The database side of that is not released yet, and the other clients do not carry ids at all so far. Only the link field is pending: the messages themselves are not new, so the entries below describe what current releases already print. + +Either way the id resolves here. Each entry below shows the message as it appears in your logs or console, explains what caused it, and gives the fix, so you can act without reading anything else first. + +You do not need a link to use this section. If you have the message text but no id, search for a distinctive phrase from it, or open the page for the area the message came from. + +## Message groups + +import CardsSection from "/src/components/CardsSection"; + +export const errorGroups = [ + { + title: "Cluster resources", + description: + "Shards that will not load, and other messages that mean the node has run out of memory, memory mappings, or disk.", + link: "/errors/cluster-resources", + icon: "fa fa-server", + }, + { + title: "Collection configuration", + description: + "Deprecated ways of describing vectors, vector indexes, and multi-vector embeddings when you create or update a collection.", + link: "/errors/collection-configuration", + icon: "fa fa-sliders", + }, + { + title: "Model integrations", + description: + "Renamed and retired embedding, multimodal, and generative model integrations.", + link: "/errors/model-integrations", + icon: "fa fa-diagram-project", + }, +]; + + + +
+ +This section is being filled in message by message. An id whose entry is not written yet lands you on this page rather than on a dead link, so if you arrived here from a message and cannot find it above, that is why. The message is still on the list, and the [support channels](/support) can help in the meantime. + +## Reading a message id + +An id looks like `core-mem001` or `py-dep024`. It has two parts: + +- The **origin**: `core` for Weaviate Database itself, and a client library short name, such as `py`, for a message raised before the request ever left your application. +- The **category and number**: a topic such as `mem` for memory, `dep` for a deprecation, or `auth` for authentication, plus a number that is unique within that origin. + +Ids are permanent. The message text and the page it points to can both change, but an id that shipped in a release keeps meaning the same thing forever, so it is safe to search for, to alert on, and to quote in a support ticket. + +Released versions of the Python client print the id **without** the origin, so what you see today is `Dep024`, not `py-dep024`. Those bare ids are permanent: they are already inside published packages and can never be reassigned. The prefixed form is what new messages use, so that the database and each client can number their own categories without colliding. Both forms lead to the same entry. + +## Questions and feedback + +import DocsFeedback from '/_includes/docs-feedback.mdx'; + + diff --git a/docs/errors/model-integrations.mdx b/docs/errors/model-integrations.mdx new file mode 100644 index 000000000..1091f1355 --- /dev/null +++ b/docs/errors/model-integrations.mdx @@ -0,0 +1,146 @@ +--- +title: Model integration messages +description: "Warnings about renamed or retired Weaviate model integrations, with the current way to configure the same provider." +image: og/default.jpg +hide_table_of_contents: true +side_panel: improve-cluster +--- + +import ClusterIdNotice from '@site/src/components/ClusterIdNotice'; + + + +Messages on this page are deprecation warnings from the Weaviate Python client, raised when a collection asks for a model integration under a name that has been renamed or retired. The provider is still supported. Only the name it is configured under has changed. + +If your message is not here, the [message index](/errors) lists the other groups. + +## Google integrations renamed from PaLM {#google-renamed-from-palm} + +Message ids: `py-dep011`, `py-dep012`, `py-dep013` + +**What you see** + +One of these warnings when you configure a collection. It prints as soon as the configuration object is built, before any request reaches Weaviate: + +```text +Dep011: text2vec-palm is deprecated and will be removed in Q2 25. Use text2vec-google instead. +Dep012: multi2vec-palm is deprecated and will be removed in Q2 25. Use multi2vec-google instead. +Dep013: generative.palm is deprecated and will be removed in Q2 25. Use generative.google instead. +``` + +They usually arrive together with a generic deprecation notice naming the same replacement. + +:::note The message's own advice is a step behind + +Do the rename the message asks for and you get a second deprecation warning, because `text2vec-google` and `generative.google` have since been split by service. The page is not contradicting your terminal. It is one step ahead of it. Skip to the service-specific form below and you are done in one edit. + +::: + +**Why it happens** + +Google retired the PaLM name and reorganized these models under Vertex AI and the Gemini API. Weaviate followed, first by renaming the integrations from `palm` to `google`, and then by splitting the Google embedding and generative integrations by service, because Vertex AI and the Gemini API authenticate differently and expose different models. + +That second split is why the message's own advice is out of date rather than wrong: it was written when `google` was the destination, and `google` has since become two. + +The removal date quoted in the message has passed, and the old names still work, so nothing is broken while you migrate. Treat them as removable at any time. + +**How to fix it** + +Choose the integration for the Google service you actually use, then configure it in place of the PaLM one. + +For text embeddings, Vertex AI needs the Google Cloud project that owns the model, and the Gemini API does not: + +```python +from weaviate.classes.config import Configure + +# Deprecated +client.collections.create( + "Article", + vectorizer_config=Configure.Vectorizer.text2vec_palm(project_id="my-project"), +) + +# Current, Vertex AI +client.collections.create( + "Article", + vector_config=Configure.Vectors.text2vec_google_vertex(project_id="my-project"), +) + +# Current, Gemini API +client.collections.create( + "Article", + vector_config=Configure.Vectors.text2vec_google_gemini(), +) +``` + +Multimodal embeddings work the same way, and this is the one that catches people out. There is a single multimodal integration rather than a renamed pair, so it looks like there is nothing to choose. There is: the configuration still selects the service. Vertex AI needs the project and the region, the Gemini API needs neither, and asking for the wrong one means being asked for a project id you do not have: + +```python +from weaviate.classes.config import Configure + +# Current, Vertex AI +client.collections.create( + "Article", + vector_config=Configure.Vectors.multi2vec_google( + project_id="my-project", + location="us-central1", + text_fields=["title"], + ), +) + +# Current, Gemini API +client.collections.create( + "Article", + vector_config=Configure.Vectors.multi2vec_google_gemini( + text_fields=["title"], + ), +) +``` + +For generative search, pick the same service you use for embeddings: + +```python +from weaviate.classes.config import Configure + +# Current, Vertex AI +client.collections.create( + "Article", + generative_config=Configure.Generative.google_vertex(project_id="my-project"), +) + +# Current, Gemini API +client.collections.create( + "Article", + generative_config=Configure.Generative.google_gemini(), +) +``` + +Do not be alarmed when the collection reports the old name back. Weaviate stores these integrations under their original module names, so a collection configured this way still reads back as `text2vec-palm`, `multi2vec-palm`, or `generative-palm`. That is the storage name, not the name you configure, and it does not need fixing. + +The collection definition is only half of the change. The header that carries your Google credentials was renamed alongside the integration, and which one is correct now depends on the service you picked. Send the current header for your service: + +| Service | Current header | Deprecated headers it replaces | +| --- | --- | --- | +| Vertex AI | `X-Goog-Vertex-Api-Key` | `X-Google-Vertex-Api-Key`, `X-Google-Api-Key`, `X-PaLM-Api-Key` | +| Gemini API | `X-Goog-Studio-Api-Key` | `X-Google-Studio-Api-Key`, `X-Google-Api-Key`, `X-PaLM-Api-Key` | + +The current headers need a server on v1.27.7, v1.26.12 or v1.25.27 at the least. Check your server version before you switch: a cluster old enough to still be running the PaLM integration may be old enough to reject the new header, and it will answer with an authentication error rather than a helpful one. On an older server, stay on the deprecated header until you upgrade. + +The deprecated headers still work everywhere, so a running application does not break the moment you change the collection definition. Where your server is new enough, move both in the same change and you will not have to come back to it. + +One thing to notice when you diff the snippets above: the current forms also move the model onto the single vector definition, which is a separate deprecation covered in [deprecated vector configuration arguments](/errors/collection-configuration#deprecated-vector-configuration). That change is not part of the Google rename. Both land in the same edit because the current builders only exist in the new shape. + +Existing collections are unaffected: the integration is recorded when the collection is created, and a collection created under the old name goes on working. Change the name in the code that creates new collections. Rewriting an existing collection would mean re-embedding its objects, which is only worth doing if you also want to change the model. + +**Learn more** + +- [Google embeddings](/weaviate/model-providers/google/embeddings#configure-the-vectorizer) +- [Google multimodal embeddings](/weaviate/model-providers/google/embeddings-multimodal#configure-the-vectorizer) +- [Google generative models](/weaviate/model-providers/google/generative#configure-collection) +- [Google API credentials and headers](/weaviate/model-providers/google/embeddings#api-credentials) +- [All Google integrations](/weaviate/model-providers/google) + +## Questions and feedback + +import DocsFeedback from '/_includes/docs-feedback.mdx'; + + diff --git a/docs/improve-your-cluster.mdx b/docs/improve-your-cluster.mdx new file mode 100644 index 000000000..aafffb705 --- /dev/null +++ b/docs/improve-your-cluster.mdx @@ -0,0 +1,81 @@ +--- +title: Improve your cluster +description: "A short checklist for making a Weaviate cluster harder to break: headroom, replication, monitoring, backups, and staying current." +image: og/default.jpg +--- + +import ClusterIdNotice from '@site/src/components/ClusterIdNotice'; + + + +Most of the messages in the [error message reference](/errors) are not really about the moment they were printed. They are the point at which a cluster that was already short on headroom, unmonitored, or behind on versions finally ran out of room. + +This page is the short version of what to change so the next one does not happen. Work through it once when you set a cluster up, and again whenever the shape of your workload changes: a new collection, a jump in tenants, or a much larger import. + +## Give the cluster headroom + +Weaviate fails safely when it runs out of a resource. Shards go read-only, or refuse to load, rather than corrupting data. That is the behavior you want, but it is still an outage, and every one of these limits is knowable in advance. + +- **Memory.** Size the node for the whole working set, not the average. Tell the Go runtime what it may use with `GOMEMLIMIT`, or set `LIMIT_RESOURCES` and let Weaviate derive it. Without one of them, the runtime assumes it can use the whole machine and gets killed by the container limit instead of collecting garbage harder. +- **Disk.** Weaviate warns above 80% disk usage and marks shards read-only above 90%, both tunable. Compaction and backups both need free space, so treat 90% as the ceiling and not the target. +- **Memory mappings.** Every shard costs kernel memory mappings, and the limit is a host setting, not a container one. Thousands of active tenants on one node will pass a distribution default. See [not enough memory mappings](/errors/cluster-resources#not-enough-memory-mappings). +- **Vector memory.** Vector indexes are the largest consumer, and quantization is the biggest single lever you have. Compression trades a small amount of recall for a large reduction in memory, and it can be enabled on a collection you already have. +- **Inactive tenants.** In a multi-tenant collection, tenants you are not serving should not be resident. Deactivating or offloading them returns memory, mappings, and file handles. + +Further reading: [resource planning](/weaviate/concepts/resources) · [environment variables](/deploy/configuration/env-vars) · [compression](/weaviate/configuration/compression) · [tenant states](/weaviate/manage-collections/tenant-states) + +## Survive the loss of a node + +A single-node cluster has no bad days, only outages. If Weaviate holds data you cannot rebuild quickly from somewhere else, it needs more than one copy. + +- **Set a replication factor above one** on collections that matter. Set it when you create the collection: raising it later is possible, but it copies data across the network at a moment you probably did not choose. +- **Use an odd number of nodes** so the cluster can still form a majority when one is lost. +- **Choose consistency levels deliberately.** Weaviate lets you pick per request, so the write path and the read path can make different trade-offs. +- **Turn on async replication** so replicas that fall behind, or that miss writes while restarting, repair themselves in the background instead of serving stale results until someone notices. + +Further reading: [replication](/deploy/configuration/replication) · [async replication](/deploy/configuration/async-rep) · [consistency](/weaviate/concepts/replication-architecture/consistency) + +## Find out before your users do + +Every resource limit above announces itself in the logs and in the metrics well before it becomes an outage. That warning is only worth having if something is reading it. + +- **Scrape the metrics endpoint.** Weaviate exposes Prometheus metrics, and there is a ready-made Grafana setup to start from. +- **Alert on the leading indicators**, not on request failures: heap usage against the limit, disk usage against the read-only threshold, and shards reporting a read-only status. +- **Collect the logs somewhere searchable.** Weaviate logs JSON by default, so the fields are queryable without parsing message text. Alert on the `action` field rather than on message wording, which changes between releases. +- **Watch the cluster node data** for shards that are not ready. It is the fastest way to tell a slow cluster from a partly broken one. + +Further reading: [monitoring](/deploy/configuration/monitoring) · [cluster status](/deploy/configuration/status#cluster-node-data) · [logging](/deploy/configuration/logging) + +## Be able to go back + +Replication protects you from losing a node. It does not protect you from a bad import, a mistaken deletion, or a schema change you want to undo, because all three replicate perfectly. + +- **Configure a backup backend** and take backups on a schedule, to storage that is not the cluster's own disk. +- **Restore one.** An untested backup is a hypothesis. Restore into a scratch cluster and query it, at least once, before you need it to work. +- **Check what a backup covers** against what you would need to rebuild, including collection configuration and inactive tenants. +- **Keep persistence on a volume that outlives the container.** A container restart should not be a data-loss event. + +Further reading: [backups](/deploy/configuration/backups) · [persistence](/deploy/configuration/persistence) + +## Stay current + +Running a version that is a year old means carrying every bug that has since been fixed, and paying for the upgrade later anyway, in one larger jump. + +- **Read the release notes** before upgrading, and the migration guide when there is one. +- **Check the known issues page** when you hit something strange. A surprising number of one-off mysteries are documented there with a version number attached. +- **Keep clients within the supported range** of the server. Client and server versions move independently, and a large gap between them is a common source of confusing errors. + +Further reading: [release notes](/weaviate/release-notes) · [known issues](/weaviate/release-notes/known-issues) · [migration guides](/deploy/migration) + +## Further resources + +- [Production readiness checklist](/deploy/production/kubernetes/production-readiness) +- [Getting to production on Kubernetes](/deploy/production/kubernetes/get-to-production) +- [Managing resources starter guide](/weaviate/starter-guides/managing-resources) +- [Error message reference](/errors) + +## Questions and feedback + +import DocsFeedback from '/_includes/docs-feedback.mdx'; + + diff --git a/netlify.toml b/netlify.toml index 2b4b5d2e6..1df570872 100644 --- a/netlify.toml +++ b/netlify.toml @@ -1250,3 +1250,462 @@ status = 301 from = "/weaviate/tutorials/query" to = "/weaviate/search" status = 301 + +### ==================== ERROR-MESSAGE REDIRECTOR (/e/) ==================== ### +# +# Stable short links carried in error and warning messages emitted by Weaviate +# core and the client libraries, e.g. "Dep004: ... see https://docs.weaviate.io/e/py-dep004". +# +# THE ID IS THE CONTRACT. THE DESTINATION IS NOT. +# A message string ships inside a released binary and can never be changed for +# users already running it. A docs URL changes every time the IA moves. So the +# message carries only the id, and the id-to-page mapping lives here. When a page +# moves or is renamed, edit the `to` below -- that is a one-line docs change +# instead of six client releases, and readers on old client versions are fixed +# retroactively. Never delete a rule; repoint it. +# +# ID FORMAT: -, lowercase. +# origin = py | ts | go | java | csharp | core +# category = auth | bat | con | dep | grpc | rbac | mem | ... +# The origin prefix exists to prevent CROSS-ORIGIN collision, and that reason +# stands on its own: the categories are semantic, not proprietary, so core and +# every client will each want a "deprecation" bucket. Core would naturally reach +# for Dep0xx, and the Python client already owns 19 of them. Without the prefix +# the second origin to allocate an id collides by default. +# +# What the prefix does NOT do is fix a WITHIN-origin collision: py-con004 would +# still name two different messages. That is a separate rule, and the Python +# client is the cautionary example for it -- ids must be ASSIGNED FROM A +# REGISTRY, not harvested from whatever prefixes a client already happens to use. +# +# LOWERCASE ONLY. Netlify matches paths case-sensitively. Messages print the id +# in mixed case for readability ("Dep004:") but MUST emit the URL lowercased +# ("/e/py-dep004"). Do not add mixed-case duplicates here; fix the emitter. +# +# Adding a row: verify the target page file exists and that the anchor is a real, +# unique heading slug (or an explicit {#anchor}, or an row id) on that +# page. Docusaurus warns rather than fails on a broken anchor, and its check only +# sees links written inside the site's own pages -- it never reads this file. So a +# bad fragment here will not fail the build. It will just 404 a user who is +# already having a bad day. +# +# DESTINATIONS ARE MIGRATING TO /errors. That section is written for exactly this +# arrival: one entry per message, with the message text, the cause and the fix on +# the page the reader lands on, no table of contents to scroll past. Ids that +# already have an entry point there. The rest still point at the nearest existing +# docs page and get repointed as their entry is written -- an approximate page +# beats no page, but a purpose-built entry beats both. + +# NO `to` VALUE IN THIS BLOCK MAY CARRY A QUERY STRING. This is the one rule +# here that is not a style preference, and it is enforced by +# _build_scripts/validate-redirects.js. Measured with `netlify dev` and +# confirmed against production docs.weaviate.io: +# +# to = "/errors/x#anchor" + request ?clusterid=U -> /errors/x?clusterid=U#anchor +# to = "/errors/x?src=py-dep011#anchor" + ?clusterid=U -> /errors/x?src=py-dep011#anchor +# +# The first line is what you want and Netlify orders the query before the +# fragment for you. The second is the trap: once a destination has a query +# string of its own, the request's own query is DROPPED, not merged. Weaviate +# appends `?clusterid=` to these links, and a `?` added here would throw +# it away silently -- no build error, no broken link, just a parameter that +# stops arriving. +# +# The message id is added instead by netlify/edge-functions/error-link-src.ts, +# which reads it from the path and sets `src=` before these rules run, which +# is why the destinations below stay plain. Read that file before changing +# anything in this block. + +## ---- Python client (weaviate-python-client) ---- + +[[redirects]] +from = "/e/py-auth001" +to = "/deploy/configuration/authentication#anonymous-access" +status = 301 + +[[redirects]] +from = "/e/py-auth005" +to = "/deploy/configuration/oidc" +status = 301 + +# py-auth006 is the runtime AuthenticationFailedError raised in +# weaviate/connect/v4.py ("No login credentials provided..."). It has no bare +# legacy id because it has never shipped with one, so it gets no alias below. +[[redirects]] +from = "/e/py-auth006" +to = "/weaviate/connections/connect-local#authentication-enabled" +status = 301 + +[[redirects]] +from = "/e/py-bat003" +to = "/weaviate/client-libraries/python/notes-best-practices#batch-sizing" +status = 301 + +[[redirects]] +from = "/e/py-bat005" +to = "/weaviate/client-libraries/python/notes-best-practices#batch-sizing" +status = 301 + +[[redirects]] +from = "/e/py-con001" +to = "/deploy/configuration/oidc#configuring-the-oidc-token-issuer" +status = 301 + +[[redirects]] +from = "/e/py-con002" +to = "/weaviate/config-refs/datatypes#date" +status = 301 + +# py-con004 is overloaded in the client: it labels BOTH the unclosed-connection +# ResourceWarning and the year-zero date warning. This points at the former, +# which is by far the more frequently hit. Renumbering the latter onto the free +# Con003 needs a client change; until that ships there is no second id to point +# at anything, so no rule is written for one. +[[redirects]] +from = "/e/py-con004" +to = "/weaviate/client-libraries/python/notes-best-practices#connection-termination" +status = 301 + +# Anchor is an row id, not a heading slug: src/components/APITable/index.jsx +# sets for every row, verbatim and case-preserving. +[[redirects]] +from = "/e/py-con005" +to = "/deploy/configuration/env-vars#GRPC_MAX_MESSAGE_SIZE" +status = 301 + +[[redirects]] +from = "/e/py-dep004" +to = "/weaviate/release-notes#version-support-policy" +status = 301 + +[[redirects]] +from = "/e/py-dep005" +to = "/weaviate/client-libraries/python#installation" +status = 301 + +# Not client-libraries/python#library-imports, the obvious-looking match: that +# heading is nested in a collapsed
("Migration guides - beta releases"). +# Docusaurus renders
React-controlled and does NOT act on the URL hash, +# so the browser's native auto-expand never fires -- the anchor resolves, the page +# returns 200, and the reader sees nothing. It is also a historical v4.4b7 beta +# migration note, a poor destination for a warning still emitted today. This +# section is current guidance and teaches the exact fix the message asks for +# (import from the submodule, not the weaviate root). +[[redirects]] +from = "/e/py-dep010" +to = "/weaviate/client-libraries/python/notes-best-practices#helper-classes" +status = 301 + +[[redirects]] +from = "/e/py-dep011" +to = "/errors/model-integrations#google-renamed-from-palm" +status = 301 + +[[redirects]] +from = "/e/py-dep012" +to = "/errors/model-integrations#google-renamed-from-palm" +status = 301 + +[[redirects]] +from = "/e/py-dep013" +to = "/errors/model-integrations#google-renamed-from-palm" +status = 301 + +[[redirects]] +from = "/e/py-dep017" +to = "/errors/collection-configuration#deprecated-vector-configuration" +status = 301 + +[[redirects]] +from = "/e/py-dep018" +to = "/weaviate/config-refs/collections#sharding" +status = 301 + +[[redirects]] +from = "/e/py-dep019" +to = "/weaviate/configuration/compression/pq-compression#pq-parameters" +status = 301 + +[[redirects]] +from = "/e/py-dep020" +to = "/weaviate/client-libraries/python/notes-best-practices#error-handling" +status = 301 + +[[redirects]] +from = "/e/py-dep021" +to = "/weaviate/manage-collections/tenant-states#tenant-states-overview" +status = 301 + +[[redirects]] +from = "/e/py-dep022" +to = "/cloud/manage-clusters/connect#connect-with-an-api-programmatically" +status = 301 + +[[redirects]] +from = "/e/py-dep023" +to = "/errors/collection-configuration#deprecated-vector-configuration" +status = 301 + +[[redirects]] +from = "/e/py-dep024" +to = "/errors/collection-configuration#deprecated-vector-configuration" +status = 301 + +[[redirects]] +from = "/e/py-dep025" +to = "/errors/collection-configuration#deprecated-vector-configuration" +status = 301 + +# py-dep026 is the second overloaded id: it labels both the named-vector syntax +# in config.add_vector() and the `encoding` argument in +# Configure.VectorIndex.MultiVectors.multi_vector(). This destination covers the +# NamedVectors -> Vectors/MultiVectors rename, which is the common denominator. +# Splitting the second message onto a free id needs a client change, the same as +# py-con004; until it ships both messages share this destination. +[[redirects]] +from = "/e/py-dep026" +to = "/errors/collection-configuration#deprecated-named-vector-syntax" +status = 301 + +[[redirects]] +from = "/e/py-dep027" +to = "/errors/collection-configuration#deprecated-multi-vector-settings" +status = 301 + +# Anchor is an explicit {#async-config} override on the "`asyncConfig` parameters" +# heading, which wins over the derived slug (asyncconfig-parameters). +[[redirects]] +from = "/e/py-dep029" +to = "/weaviate/config-refs/collections#async-config" +status = 301 + +[[redirects]] +from = "/e/py-rbac001" +to = "/weaviate/configuration/rbac#available-permissions" +status = 301 + +## ---- Weaviate core ---- + +[[redirects]] +from = "/e/core-mem001" +to = "/errors/cluster-resources#not-enough-memory-mappings" +status = 301 + +## ---- FROZEN: bare legacy ids shipped by weaviate-python-client ---- +# +# These 32 ids ship TODAY inside released weaviate-client wheels, printed at the +# front of the warning text ("Dep004: You are connected to Weaviate ..."). They +# predate the - prefix and can NEVER be renamed or removed: the strings +# are baked into versions users are still running, and will be for years. Every +# rule below is permanent. This list is closed as of the client release that +# adopts the prefixed form -- do not add to it. +# +# Two of the 32 are overloaded inside the client itself: con004 and dep026 each +# label two different messages. Note what that is and is not evidence for. It is +# NOT the reason for the origin prefix -- a prefix cannot help here, since +# py-con004 is just as ambiguous as con004. It is the reason ids must be assigned +# from a registry rather than harvested ad hoc from the prefixes already in use, +# which is how two messages ended up sharing a number in the first place. + +[[redirects]] +from = "/e/auth001" +to = "/e/py-auth001" +status = 301 + +[[redirects]] +from = "/e/auth005" +to = "/e/py-auth005" +status = 301 + +[[redirects]] +from = "/e/bat003" +to = "/e/py-bat003" +status = 301 + +[[redirects]] +from = "/e/bat005" +to = "/e/py-bat005" +status = 301 + +[[redirects]] +from = "/e/con001" +to = "/e/py-con001" +status = 301 + +[[redirects]] +from = "/e/con002" +to = "/e/py-con002" +status = 301 + +[[redirects]] +from = "/e/con004" +to = "/e/py-con004" +status = 301 + +[[redirects]] +from = "/e/con005" +to = "/e/py-con005" +status = 301 + +[[redirects]] +from = "/e/dep004" +to = "/e/py-dep004" +status = 301 + +[[redirects]] +from = "/e/dep005" +to = "/e/py-dep005" +status = 301 + +[[redirects]] +from = "/e/dep010" +to = "/e/py-dep010" +status = 301 + +[[redirects]] +from = "/e/dep011" +to = "/e/py-dep011" +status = 301 + +[[redirects]] +from = "/e/dep012" +to = "/e/py-dep012" +status = 301 + +[[redirects]] +from = "/e/dep013" +to = "/e/py-dep013" +status = 301 + +[[redirects]] +from = "/e/dep017" +to = "/e/py-dep017" +status = 301 + +[[redirects]] +from = "/e/dep018" +to = "/e/py-dep018" +status = 301 + +[[redirects]] +from = "/e/dep019" +to = "/e/py-dep019" +status = 301 + +[[redirects]] +from = "/e/dep020" +to = "/e/py-dep020" +status = 301 + +[[redirects]] +from = "/e/dep021" +to = "/e/py-dep021" +status = 301 + +[[redirects]] +from = "/e/dep022" +to = "/e/py-dep022" +status = 301 + +[[redirects]] +from = "/e/dep023" +to = "/e/py-dep023" +status = 301 + +[[redirects]] +from = "/e/dep024" +to = "/e/py-dep024" +status = 301 + +[[redirects]] +from = "/e/dep025" +to = "/e/py-dep025" +status = 301 + +[[redirects]] +from = "/e/dep026" +to = "/e/py-dep026" +status = 301 + +[[redirects]] +from = "/e/dep027" +to = "/e/py-dep027" +status = 301 + +[[redirects]] +from = "/e/dep029" +to = "/e/py-dep029" +status = 301 + +[[redirects]] +from = "/e/rbac001" +to = "/e/py-rbac001" +status = 301 + +# The five below are frozen aliases too, but their canonical py-* id has NO +# destination rule yet: the docs do not currently cover what these messages tell +# the user to do, and py-dep028's only candidate page still demonstrates the very +# argument the warning deprecates. So they fall through to the catch-all and land +# on the /errors index until the content exists. Do NOT "fix" that by pointing +# them at an approximately-related page; a wrong page in an error message is +# worse than no page. Write the content, add the single /e/py- rule above, +# and these start working. + +[[redirects]] +from = "/e/auth002" +to = "/e/py-auth002" +status = 301 + +[[redirects]] +from = "/e/auth003" +to = "/e/py-auth003" +status = 301 + +[[redirects]] +from = "/e/auth004" +to = "/e/py-auth004" +status = 301 + +[[redirects]] +from = "/e/dep028" +to = "/e/py-dep028" +status = 301 + +[[redirects]] +from = "/e/grpc002" +to = "/e/py-grpc002" +status = 301 + +## ---- CATCH-ALL: an id with no row of its own ---- +# +# MUST STAY LAST IN THE FILE. Netlify matching is first-match, so every specific +# rule above still wins; this only fires for an id nothing else claimed. +# +# Two populations land here. An id shipped by a release that is ahead of the docs, +# and the handful of frozen aliases whose canonical `py-*` id deliberately has no +# destination yet because the content does not exist (auth002, auth003, auth004, +# dep028, grpc002). Both used to get a 404. The /errors index is a strictly +# better answer: it explains what an id is, lists the groups, and says outright +# that the section is still being filled in. +# +# This does NOT make it safe to skip writing a rule. A reader who lands on the +# index has to find their own message; a reader sent to an entry has already been +# answered. The catch-all is a floor, not a destination. +# +# 302, AND IT IS THE ONLY RULE HERE THAT IS. Every specific rule above is a 301 +# because its destination is a promise: an id means one thing forever, so caching +# it in a browser for good is exactly right. This rule is the opposite. It fires +# for ids whose entry is not written YET, so its destination changes the moment +# one is -- and a 301 already in someone's browser cache would keep sending them +# to the index long after their id got a real entry, with nothing we could deploy +# to fix it. Do not "make it consistent" with the rules above. +# +# ENFORCED, not merely requested: _build_scripts/validate-redirects.js checks +# that this rule is last and 302, that every rule above it is 301 and lowercase +# and unique, and that every /errors destination resolves to a real page and a +# real {#anchor}. It runs in CI on every PR and needs no build: +# node _build_scripts/validate-redirects.js +[[redirects]] +from = "/e/*" +to = "/errors" +status = 302 diff --git a/netlify/edge-functions/error-link-src.ts b/netlify/edge-functions/error-link-src.ts new file mode 100644 index 000000000..61577ed12 --- /dev/null +++ b/netlify/edge-functions/error-link-src.ts @@ -0,0 +1,53 @@ +/** + * Sets `src=` on /e/ requests before the redirect rules run. + * + * Reads the id out of the request path, adds it to the query string, and + * rewrites. Any query the request already had is preserved alongside it. + * + * WHY IT IS DONE HERE AND NOT IN THE `to` VALUES + * ---------------------------------------------- + * Netlify forwards an incoming query string to a redirect destination ONLY when + * the destination has no query string of its own. Write + * `to = "/errors/x?src=py-dep011#anchor"` and the query the request arrived with + * is silently DROPPED -- no build error, no broken link. Measured with + * `netlify dev` and confirmed against production docs.weaviate.io. So every `to` + * in the /e/ block stays plain and the parameter is added here instead; + * _build_scripts/validate-redirects.js enforces the plain `to` values. + * + * ORDERING + * -------- + * Edge functions run BEFORE the redirect rules (measured: a function and a 301 + * on the same path -> the function answers and the 301 never fires), and a + * rewrite re-enters the redirect engine without re-running this function, so + * rewriting to the same path with an extra param terminates rather than looping. + * + * `onError: "bypass"` skips this function if it throws, and the request falls + * through to the plain redirect rules. Without it a thrown error is a 500 -- + * measured -- on a link that is printed inside an error message. Do not remove + * it. + */ + +export const config = { + path: "/e/*", + onError: "bypass", +}; + +// Same shape the redirector's ids are required to have: lowercase +// -. Anything else is left alone rather than reflected +// into a URL, so a crafted path cannot put arbitrary text in the query string. +const ID = /^[a-z0-9]+(?:-[a-z0-9]+)*$/; + +export default async (request: Request, context: any) => { + const url = new URL(request.url); + + // A frozen legacy alias redirects /e/dep011 -> /e/py-dep011, so this can run + // twice for a single click, the second time on a request that already carries + // the parameter. The value set on the first hop stands. + if (url.searchParams.has("src")) return context.next(); + + const id = url.pathname.slice("/e/".length).replace(/\/+$/, ""); + if (!id || id.length > 64 || !ID.test(id)) return context.next(); + + url.searchParams.set("src", id); + return context.rewrite(url.toString()); +}; diff --git a/package.json b/package.json index 5cf18108b..2f626e1c3 100644 --- a/package.json +++ b/package.json @@ -9,6 +9,7 @@ "build": "docusaurus build", "build-dev": "docusaurus build --config docusaurus.dev.config.js --out-dir build.dev", "validate-links-dev": "node ./_build_scripts/validate-links-pr.js", + "validate-redirects": "node ./_build_scripts/validate-redirects.js", "swizzle": "docusaurus swizzle", "deploy": "docusaurus deploy", "clear": "docusaurus clear", diff --git a/src/components/ClusterIdNotice/index.jsx b/src/components/ClusterIdNotice/index.jsx new file mode 100644 index 000000000..493fd846f --- /dev/null +++ b/src/components/ClusterIdNotice/index.jsx @@ -0,0 +1,67 @@ +import React, { useEffect, useState } from "react"; +import { useLocation } from "@docusaurus/router"; +import styles from "./styles.module.scss"; + +/** + * Shows the cluster identity a reader arrived with, when they arrived with one. + * + * Weaviate can append `?clusterid=` to the docs links it prints, so the + * reader lands holding it. This renders it. + * + * Rendered on the four /errors entry pages, which is where a message link + * actually lands, and on /improve-your-cluster, which is one link further on. + * The redirect rules in netlify.toml carry the parameter to all five; the + * component is page-agnostic and needs nothing from the page it sits on. + * + * ABSENT IS THE NORMAL CASE, NOT AN ERROR. Weaviate's ClusterID() returns an + * empty string until the raft leader has committed an identity, a cluster may + * never have one at all, and plenty of readers just type the URL. So every one + * of absent, empty and malformed renders nothing at all -- no placeholder, no + * warning, no "invalid cluster id". A reader who never had an id is not having + * a problem, and telling them otherwise is noise on a page they came to for + * something else. + * + * IT NEVER LOOKS THE ID UP. The check is a format check and nothing more: no + * request, no 404 on an unknown id, no difference in what renders between an id + * that exists and one that does not. Anything else would be an unauthenticated + * oracle for guessing cluster identities. It is also why the id is validated + * before it is rendered rather than passed through -- an arbitrary path segment + * should never reach the DOM, React escaping or not. + * + * NO CANONICAL TAG IS NEEDED HERE. Docusaurus already emits a route-derived + * `` with no query string; verified against production, + * where /weaviate/release-notes/known-issues?clusterid=abc-123 still serves + * canonical https://docs.weaviate.io/weaviate/release-notes/known-issues. Do + * not add a second one -- two canonical tags on a page are worth less than one. + */ + +// Canonical 8-4-4-4-12 hex, version-agnostic on purpose. Weaviate mints a v7 +// and falls back to v4 when the monotonic-random source fails, so a regex that +// pinned the version nibble would reject exactly the ids born on a bad day. +const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; + +export default function ClusterIdNotice() { + const location = useLocation(); + const [clusterId, setClusterId] = useState(null); + + // Read after mount rather than during render. The server-rendered HTML is + // built without a query string, so resolving this inline would make the first + // client render disagree with it and trip a hydration mismatch. + useEffect(() => { + const raw = new URLSearchParams(location.search).get("clusterid"); + const value = (raw || "").trim(); + setClusterId(UUID.test(value) ? value.toLowerCase() : null); + }, [location.search]); + + if (!clusterId) return null; + + return ( + + ); +} diff --git a/src/components/ClusterIdNotice/styles.module.scss b/src/components/ClusterIdNotice/styles.module.scss new file mode 100644 index 000000000..c6de12592 --- /dev/null +++ b/src/components/ClusterIdNotice/styles.module.scss @@ -0,0 +1,31 @@ +/* src/components/ClusterIdNotice/styles.module.scss */ + +.notice { + border: 1px solid var(--ifm-color-emphasis-300); + border-left: 3px solid var(--ifm-color-primary); + border-radius: 6px; + padding: 0.9rem 1rem; + background: var(--ifm-background-surface-color); + margin: 0 0 1.5rem; +} + +.label { + margin: 0 0 0.3rem; + font-size: 0.7rem; + font-weight: 700; + letter-spacing: 0.08em; + text-transform: uppercase; + color: var(--ifm-color-emphasis-600); +} + +.value { + margin: 0 0 0.5rem; + word-break: break-all; +} + +.body { + margin: 0; + font-size: 0.85rem; + line-height: 1.5; + color: var(--ifm-color-emphasis-800); +} diff --git a/src/components/ErrorSidePanel/index.jsx b/src/components/ErrorSidePanel/index.jsx new file mode 100644 index 000000000..4a834a435 --- /dev/null +++ b/src/components/ErrorSidePanel/index.jsx @@ -0,0 +1,35 @@ +import React from "react"; +import Link from "@docusaurus/Link"; +import styles from "./styles.module.scss"; + +/** + * Right-hand panel for the /errors section. + * + * Error pages set `hide_table_of_contents: true` (readers arrive at one + * anchor from a message link, so a list of the other anchors is noise) and + * `side_panel: improve-cluster`. src/theme/DocItem/Layout renders this in the + * column the table of contents would otherwise occupy. Nothing else on the + * site sets `side_panel`, so no normal docs page is affected. + */ +export default function ErrorSidePanel() { + return ( + + ); +} diff --git a/src/components/ErrorSidePanel/styles.module.scss b/src/components/ErrorSidePanel/styles.module.scss new file mode 100644 index 000000000..8923787d9 --- /dev/null +++ b/src/components/ErrorSidePanel/styles.module.scss @@ -0,0 +1,47 @@ +/* src/components/ErrorSidePanel/styles.module.scss */ + +.panel { + border: 1px solid var(--ifm-color-emphasis-300); + border-radius: 8px; + padding: 1.1rem 1.15rem; + background: var(--ifm-background-surface-color); + margin-top: 1.5rem; +} + +.eyebrow { + margin: 0 0 0.2rem; + font-size: 0.7rem; + font-weight: 700; + letter-spacing: 0.08em; + text-transform: uppercase; + color: var(--ifm-color-emphasis-600); +} + +.title { + margin: 0 0 0.6rem; + font-size: 1.05rem; + line-height: 1.3; +} + +.body { + margin: 0 0 1rem; + font-size: 0.85rem; + line-height: 1.5; + color: var(--ifm-color-emphasis-800); +} + +.cta { + display: inline-block; + padding: 0.45rem 0.9rem; + border-radius: 6px; + font-size: 0.85rem; + font-weight: 600; + color: var(--ifm-color-primary-contrast-background); + background: var(--ifm-color-primary); + + &:hover { + color: var(--ifm-color-primary-contrast-background); + background: var(--ifm-color-primary-dark); + text-decoration: none; + } +} diff --git a/src/theme/DocItem/Layout/index.js b/src/theme/DocItem/Layout/index.js index 071ba741d..1a53fbe63 100644 --- a/src/theme/DocItem/Layout/index.js +++ b/src/theme/DocItem/Layout/index.js @@ -18,8 +18,19 @@ import styles from "./styles.module.css"; import FeedbackComponent from "@site/src/components/Feedback"; import PageRatingWidget from "@site/src/components/PageRatingWidget"; import ContextualMenu from "@site/src/components/ContextualMenu"; +import ErrorSidePanel from "@site/src/components/ErrorSidePanel"; /* ---- END: Customizations ---- */ +// Opt-in replacement for the right-hand column, keyed on the `side_panel` +// frontmatter field. Only the /errors pages set it: they hide the table of +// contents (a reader arriving from an error message wants one anchor, not a +// list of the others) and put a pointer to /improve-your-cluster there +// instead. A page that does not set `side_panel` takes the original path +// unchanged. +const SIDE_PANELS = { + "improve-cluster": ErrorSidePanel, +}; + // Emit a schema.org FAQPage JSON-LD block when the page's frontmatter declares // a `faq:` list. Google reads this for entity understanding; rich-result FAQ // snippets themselves are restricted post-2024 so don't expect a SERP card. @@ -81,11 +92,24 @@ export default function DocItemLayout({ children }) { const showMobileFeedback = !docTOC.hidden && !docTOC.desktop && docTOC.mobile && feedbackEnabled; + // Same viewport rule the desktop TOC uses, so the panel and the TOC can never + // both claim the column. + const SidePanel = SIDE_PANELS[frontMatter.side_panel]; + const showSidePanel = + Boolean(SidePanel) && + !docTOC.desktop && + (windowSize === "desktop" || windowSize === "ssr"); + return ( <>
-
+
@@ -107,13 +131,14 @@ export default function DocItemLayout({ children }) {
{/* ---- REMOVED: Feedback component from main column ---- */}
- {docTOC.desktop && ( + {(docTOC.desktop || showSidePanel) && (
{/* ---- START: Customizations ---- */}
{docTOC.desktop} + {showSidePanel && } {/* TODO: Temporarily hidden while debugging env vars */} - + {docTOC.desktop && }
{/* Feedback component back in TOC column */} {showFeedback && (