diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index f3cfb9a..9baf328 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -18,13 +18,46 @@ example README, package README and user-facing command text.
document fields, CLI commands, what each pack does with intent, and the
published standards' files) are rendered by components in
`components/reference/`. Change the code, not the page.
-- Every page belongs to one of seven areas, each a folder whose `meta.json`
- says `"root": true`. A folder in brackets, such as `how-to/(build)/`,
- sets a page's area without changing its URL. A new page goes in its area's
- folder and `meta.json`, and its URL into `test/page-urls.txt`; a page never
- moves to a new URL without a redirect in `worker/redirects.js`. A section
- that moves or is renamed keeps its old id with ``,
- because a browser cannot redirect a fragment.
+- Each platform has a site of its own, chosen with the picker in the header:
+ `synapse/`, `databricks/`, `adf/` and `duckdb/`, each with the same
+ sections (Get started, Guides, Standards, Connectors, then its own Concepts
+ and Reference pages). A section is a folder in brackets, such as
+ `synapse/(guides)/`, whose `meta.json` says `"root": true`; the brackets set
+ the tab without changing the URL, `/docs/synapse/`. The pages that
+ read the same everywhere (definitions, YAML, the command line, the editor,
+ errors) are in `concepts/` and `(reference)/`, one address each, with no
+ platform's text in them. `lib/platforms.ts` says which section and
+ platforms each page belongs to; `test/areas.test.ts` and
+ `test/platform-paths.test.ts` pin them. A page never moves to a new URL
+ without a redirect in `worker/redirects.js` or a page at the old URL that
+ lists its versions; a section that moves or is renamed keeps its old id
+ with ``, because a browser cannot redirect a fragment.
+- A page several platforms show is written once, in `content/shared/`, and
+ each of those platforms has a stub: front matter and one ``,
+ nothing else. **Edit the shared file, never a stub**, and read a change to a
+ shared file on every platform that shows it. Text for some platforms goes
+ in `…`; a link to a page on the
+ reader's own platform is `/docs/~/`, and a component's
+ `platform="~"` is the page's platform. The build keeps only the page's
+ platform's text (`lib/remark-only.ts`), so its contents and search hold
+ nothing else. `test/stubs.test.ts` fails when a stub holds text of its own
+ or a product-wide page holds ``.
+- **A platform's docs name no other platform**, in prose, YAML, commands,
+ outputs, alt text, links or examples. Each example on a platform's page is
+ that platform's own project; an example named `-~` is the page's
+ platform's, such as `data-products-synapse`. Data Factory into another
+ platform is a combined site of its own, `/docs/adf-`, listed
+ under **Combined** in the picker, and its pages may name its two
+ platforms. `test/platform-leak.test.ts` checks the pages' text and
+ `pnpm check:links` the built pages, components included; nothing is
+ allowed. Product-wide pages (Concepts, Reference, the home page) may name
+ every platform. Keep `` blocks out of list items: Prettier
+ re-indents them there.
+- Connector pages are generated per platform: a connector's notes are one
+ file in `content/shared/connectors/`, and `pnpm stubs` writes each
+ platform's stubs, its Connectors `meta.json` and the old
+ `/docs/connectors/` pages from the connector facts.
+ `test/connector-sites.test.ts` fails when they are out of date.
- A code block says what it is. `sh` fences are commands to run; `yaml`, `json`,
`csv` and `dotenv` fences and `` files are files to write; an
untitled `text` fence is what Ingestron prints; an `` under `build/`
diff --git a/app/api/search/route.ts b/app/api/search/route.ts
index ea164e1..4d1f4f9 100644
--- a/app/api/search/route.ts
+++ b/app/api/search/route.ts
@@ -1,12 +1,21 @@
import type { StructuredData } from "fumadocs-core/mdx-plugins";
import { createFromSource } from "fumadocs-core/search/server";
import { extraStructure, searchTitle } from "@/lib/search";
-import { source } from "@/lib/source";
+import { searchBreadcrumbs } from "@/lib/search-breadcrumbs";
+import { platformIds, platformOfPath } from "@/lib/platforms";
+import { movedGuide, source } from "@/lib/source";
// Exported once at build time as a static search index. Reference rendered
// from code is added to its page's entries, so error codes and settings are found.
+// A platform page is tagged with its platform and a shared page with every
+// platform, so the dialog's platform filter keeps the shared sections. A
+// guide's old URL, which only links to its versions, is left out.
export const revalidate = false;
-export const { staticGET: GET } = createFromSource(source, {
+const indexed = {
+ ...source,
+ getPages: () => source.getPages().filter((page) => !movedGuide(page.path)),
+};
+export const { staticGET: GET } = createFromSource(indexed as typeof source, {
language: "english",
buildIndex(page) {
const own = page.data.structuredData as StructuredData;
@@ -16,6 +25,9 @@ export const { staticGET: GET } = createFromSource(source, {
description: page.data.description,
url: page.url,
id: page.url,
+ tag: platformOfPath(page.url) ?? [...platformIds],
+ // A platform's page starts with its platform, so the copies of a shared page differ.
+ breadcrumbs: searchBreadcrumbs(page.url, source.getPageTree()),
structuredData: {
headings: [...own.headings, ...extra.headings],
contents: [...own.contents, ...extra.contents],
diff --git a/app/content.css b/app/content.css
index 81f7de8..5fb6ced 100644
--- a/app/content.css
+++ b/app/content.css
@@ -225,3 +225,20 @@ pre code > .line {
display: block;
margin-top: 0.3rem;
}
+
+/* Above a platform page: which platform it is for, and the same page on the others. */
+/* A guide's old URL: one line per platform version. */
+.platform-chooser {
+ list-style: none;
+ padding: 0;
+}
+.platform-chooser li {
+ margin-block: 0.5rem;
+}
+.platform-chooser a {
+ font-weight: 600;
+}
+.platform-continue {
+ margin-block: 0 1rem;
+ font-weight: 600;
+}
diff --git a/app/docs/(areas)/[...slug]/page.tsx b/app/docs/(areas)/[...slug]/page.tsx
index 38b296e..c16f2b4 100644
--- a/app/docs/(areas)/[...slug]/page.tsx
+++ b/app/docs/(areas)/[...slug]/page.tsx
@@ -5,25 +5,30 @@ import { getMDXComponents } from "@/components/mdx";
import { RelatedSetup } from "@/components/reference/related-setup";
import { catalogueToc, connectorToc } from "@/lib/connector-page";
import { connectorEntries } from "@/lib/reference";
-import { source } from "@/lib/source";
+import { platformOfPath, type Platform } from "@/lib/platforms";
+import { movedGuide, source } from "@/lib/source";
export default async function Page({ params }: { params: Promise<{ slug: string[] }> }) {
const { slug } = await params;
const page = source.getPage(slug);
if (!page) notFound();
const MDX = page.data.body;
- const id = slug[0] === "connectors" && slug.length === 2 ? slug[1] : undefined;
+ // A platform's connector page, /docs//connectors/, and its catalogue.
+ const platform = platformOfPath(slug.join("/")) as Platform["id"] | undefined;
+ const id = platform && slug[1] === "connectors" && slug.length === 3 ? slug[2] : undefined;
const connector = id && connectorEntries(id).length ? id : undefined;
- const related = connector ? : undefined;
+ const related = connector ? : undefined;
// A connector page's headings come from its component, so its contents are built here.
const toc = connector
- ? connectorToc(connector, page.data.toc)
- : slug.join("/") === "connectors"
- ? catalogueToc(page.data.toc)
+ ? connectorToc(connector, page.data.toc, platform)
+ : platform && slug.length === 2 && slug[1] === "connectors"
+ ? catalogueToc(page.data.toc, platform)
: page.data.toc;
return (
// The breadcrumb starts at the page's area, such as "Connectors" on a connector's page.
// tabIndex: the skip link focuses the article, so the next Tab is in it.
[
+ platform.id,
+ connectorsFor(platform.id).map((id) => ({
+ id,
+ title: connectorEntries(id)[0]!.connector.title,
+ })),
+ ]),
+ );
return (
<>
{/* The first stop on the page; the article it opens is #nd-page. */}
,
+ }}
tree={tree}
tabs={tabs}
tabMode="navbar"
- sidebar={{ components: { Item: SidebarPageItem } }}
+ sidebar={{
+ components: { Item: SidebarPageItem },
+ banner: (
+ <>
+
+
+ >
+ ),
+ }}
>
+ ({ url, urls: [...(urls ?? [])] }))} />
{children}
diff --git a/app/docs/(hub)/layout.tsx b/app/docs/(hub)/layout.tsx
index 4b3361f..d2d33de 100644
--- a/app/docs/(hub)/layout.tsx
+++ b/app/docs/(hub)/layout.tsx
@@ -1,26 +1,25 @@
import type { ReactNode } from "react";
import { HomeLayout } from "fumadocs-ui/layouts/home";
-import { getLayoutTabs } from "fumadocs-ui/layouts/shared";
import { SkipLink } from "@/components/keyboard";
import { baseOptions } from "@/lib/layout.shared";
-import { source } from "@/lib/source";
+import { platforms } from "@/lib/platforms";
// The home page is the hub: no sidebar and no contents, only the header. Where
-// the other pages have their tab bar, its header has an Areas menu: seven
-// links in a row would not fit beside the search at a laptop's width.
+// the other pages have the platform picker and its tab bar, its header has a
+// Platforms menu: each platform's docs, with the one line its card shows.
export default function Layout({ children }: { children: ReactNode }) {
const options = baseOptions();
- const areas = getLayoutTabs(source.getPageTree()).map((tab) => ({
- text: tab.title,
- description: tab.description,
- url: tab.url,
+ const items = platforms.map((platform) => ({
+ text: platform.title,
+ description: platform.summary,
+ url: `/docs/${platform.id}`,
}));
return (
<>
{children}
diff --git a/app/docs/(hub)/page.tsx b/app/docs/(hub)/page.tsx
index 13e6671..5dec494 100644
--- a/app/docs/(hub)/page.tsx
+++ b/app/docs/(hub)/page.tsx
@@ -3,15 +3,18 @@ import { notFound } from "next/navigation";
import { getMDXComponents } from "@/components/mdx";
import { AreaCards } from "@/components/hub";
import { HubSearch } from "@/components/hub-search";
-import { areaCards } from "@/lib/areas";
+import { areaCards, combinedAreas, platformAreas } from "@/lib/areas";
+import { ContinueWithPlatform } from "@/components/platform-choice";
import { source } from "@/lib/source";
-// The home page, /docs: search, a card for each area, then the two ways in
-// and the value story from content/docs/index.mdx.
+// The home page, /docs: search, a card for each platform, then the
+// product-wide areas, then the ways in and the value story from
+// content/docs/index.mdx.
export default function Hub() {
const page = source.getPage([]);
if (!page) notFound();
const MDX = page.data.body;
+ const cards = areaCards(source.getPageTree());
return (
// The skip link's target; tabIndex so the next Tab is in the content.
@@ -23,10 +26,33 @@ export default function Hub() {
-
-
Choose an area
-
Each area has its own short menu.
-
+
+
Choose your platform
+
+ Each platform has its own docs: getting started, guides, standards and connectors, with
+ nothing that does not apply to it. Not sure which?{" "}
+ Compare them, or try a sample on your
+ laptop with DuckDB: no cloud account needed.
+
+
+ platformAreas.includes(card.url))} />
+
Combined
+
+ Azure Data Factory copies the sources, and another platform keeps the tables.
+
+ combinedAreas.includes(card.url))} />
+
+
+
For every platform
+
+ Definitions, YAML, the command line and the VS Code extension work the same on each
+ platform.
+
diff --git a/app/hub.css b/app/hub.css
index cbeaeaa..b3e72ad 100644
--- a/app/hub.css
+++ b/app/hub.css
@@ -144,6 +144,13 @@
font-weight: 750;
letter-spacing: -0.02em;
}
+/* The combinations, under the platforms. */
+.hub-body > section > h3 {
+ margin: 2rem 0 0.3rem;
+ color: var(--docs-heading);
+ font-size: 1.125rem;
+ font-weight: 700;
+}
.hub-lead {
margin: 0 0 1.25rem;
color: var(--color-fd-muted-foreground);
@@ -260,3 +267,8 @@
margin-top: 0;
font-size: 1.1rem;
}
+
+/* The platform cards, then everything else: a gap between the two groups. */
+.hub-body > section + section {
+ margin-top: 3rem;
+}
diff --git a/app/layout.tsx b/app/layout.tsx
index 2b4b095..782cfb8 100644
--- a/app/layout.tsx
+++ b/app/layout.tsx
@@ -3,6 +3,8 @@ import type { ReactNode } from "react";
import { preload } from "react-dom";
import { RootProvider } from "fumadocs-ui/provider/next";
import { SearchFocusReturn } from "@/components/keyboard";
+import { PlatformSearchDialog } from "@/components/platform-search";
+import { siteScript } from "@/lib/site-script";
import "./global.css";
@@ -36,8 +38,15 @@ export default function Layout({ children }: { children: ReactNode }) {
preload("/fonts/manrope-variable.woff2", { as: "font", type: "font/woff2", crossOrigin: "" });
return (
+
+ {/* Before the header paints: which platform's tabs to show
+ (lib/site-script.ts). Here, in the root layout, the server
+ renders it once; a page change does not render it again, and
+ PlatformMemory marks each page the reader moves to. */}
+
+
-
+
{children}
diff --git a/app/navigation.css b/app/navigation.css
index ff4a4b6..b28acfc 100644
--- a/app/navigation.css
+++ b/app/navigation.css
@@ -31,3 +31,187 @@
.doc-links strong span {
color: var(--color-fd-primary);
}
+
+/* The connector picker at the top of a platform's Connectors sidebar. */
+.connector-picker {
+ display: grid;
+ gap: 0.375rem;
+ margin-block-end: 0.5rem;
+}
+.connector-picker label {
+ font-size: 0.8125rem;
+ font-weight: 500;
+}
+.connector-picker input {
+ width: 100%;
+ border: 1px solid var(--color-fd-border);
+ border-radius: 0.5rem;
+ background: var(--color-fd-background);
+ color: var(--color-fd-foreground);
+ padding: 0.375rem 0.625rem;
+ font-size: 0.875rem;
+}
+.connector-picker input:focus-visible {
+ outline: 2px solid var(--color-fd-ring);
+ outline-offset: 1px;
+}
+.connector-picker ul {
+ display: grid;
+ gap: 0.125rem;
+ font-size: 0.875rem;
+}
+.connector-picker a {
+ display: block;
+ border-radius: 0.375rem;
+ padding: 0.25rem 0.5rem;
+ color: var(--color-fd-muted-foreground);
+}
+.connector-picker a:hover,
+.connector-picker a:focus-visible {
+ background: var(--color-fd-accent);
+ color: var(--color-fd-accent-foreground);
+}
+.connector-picker-none {
+ padding: 0.25rem 0.5rem;
+ color: var(--color-fd-muted-foreground);
+}
+
+/* The tab row, and the phone menu's section list, hold every platform's own
+ sections; , set before the first paint
+ (lib/site-script.ts), shows one platform's. With no platform chosen, only
+ Concepts and Reference show. */
+:root:not([data-site]) a[data-platform],
+:root[data-site="synapse"] a[data-platform]:not([data-platform="synapse"]),
+:root[data-site="databricks"] a[data-platform]:not([data-platform="databricks"]),
+:root[data-site="adf"] a[data-platform]:not([data-platform="adf"]),
+:root[data-site="duckdb"] a[data-platform]:not([data-platform="duckdb"]) {
+ display: none;
+}
+
+/* The platform picker after the logo: the site's main switch. */
+.platform-picker {
+ position: relative;
+ margin-inline-start: 1rem;
+}
+.platform-picker-button {
+ display: inline-flex;
+ align-items: center;
+ gap: 0.375rem;
+ max-width: 13rem;
+ border: 1px solid var(--color-fd-border);
+ border-radius: 0.5rem;
+ background: var(--color-fd-secondary);
+ color: var(--color-fd-secondary-foreground);
+ padding: 0.25rem 0.625rem;
+ font-size: 0.875rem;
+ font-weight: 500;
+}
+.platform-picker-button span {
+ overflow: hidden;
+ text-overflow: ellipsis;
+ white-space: nowrap;
+}
+.platform-picker-button:hover,
+.platform-picker-button[aria-expanded="true"] {
+ background: var(--color-fd-accent);
+ color: var(--color-fd-accent-foreground);
+}
+.platform-picker-button:focus-visible,
+.platform-picker-list button:focus-visible,
+.platform-picker-list input:focus-visible {
+ outline: 2px solid var(--color-fd-ring);
+ outline-offset: 1px;
+}
+.platform-picker-list {
+ position: fixed;
+ z-index: 60;
+ display: grid;
+ gap: 0.125rem;
+ max-height: min(28rem, 70vh);
+ overflow-y: auto;
+ border: 1px solid var(--color-fd-border);
+ border-radius: 0.75rem;
+ background: var(--color-fd-popover);
+ color: var(--color-fd-popover-foreground);
+ padding: 0.375rem;
+ box-shadow: 0 8px 24px rgb(0 0 0 / 0.12);
+}
+.platform-picker-list button {
+ display: grid;
+ gap: 0.125rem;
+ width: 100%;
+ border-radius: 0.5rem;
+ padding: 0.5rem 0.625rem;
+ text-align: start;
+}
+.platform-picker-list button:hover,
+.platform-picker-list button:focus-visible {
+ background: var(--color-fd-accent);
+ color: var(--color-fd-accent-foreground);
+}
+.platform-picker-list button[aria-current="true"] strong {
+ color: var(--color-fd-primary);
+}
+.platform-picker-list strong {
+ font-size: 0.875rem;
+ font-weight: 600;
+}
+.platform-picker-list span {
+ font-size: 0.8125rem;
+ color: var(--color-fd-muted-foreground);
+}
+.platform-picker-list input {
+ width: 100%;
+ border: 1px solid var(--color-fd-border);
+ border-radius: 0.5rem;
+ background: var(--color-fd-background);
+ padding: 0.375rem 0.625rem;
+ font-size: 0.875rem;
+}
+.platform-picker-group,
+.platform-picker-none {
+ padding: 0.375rem 0.625rem 0.125rem;
+ font-size: 0.75rem;
+ font-weight: 600;
+ color: var(--color-fd-muted-foreground);
+}
+/* The line before the combinations. */
+.platform-picker-separator {
+ margin: 0.375rem 0.25rem;
+ border: 0;
+ border-top: 1px solid var(--color-fd-border);
+}
+/* On a phone the header is narrow: the logo's label goes, the picker stays. */
+@media (max-width: 40rem) {
+ .platform-picker {
+ margin-inline-start: 0.5rem;
+ }
+ .platform-picker-button {
+ max-width: 9.5rem;
+ }
+}
+
+/* Above a Concepts or Reference sidebar: the section's other half. */
+.section-companion {
+ margin-block-end: 0.75rem;
+ border-block-end: 1px solid var(--color-fd-border);
+ padding-block-end: 0.75rem;
+ font-size: 0.875rem;
+}
+.section-companion p {
+ padding-inline: 0.5rem;
+ margin-block-end: 0.25rem;
+ font-weight: 500;
+ color: var(--color-fd-foreground);
+}
+.section-companion a {
+ display: block;
+ border-radius: 0.5rem;
+ padding: 0.375rem 0.5rem;
+ color: var(--color-fd-muted-foreground);
+}
+.section-companion a:hover,
+.section-companion a:focus-visible {
+ background: var(--color-fd-accent);
+ color: var(--color-fd-accent-foreground);
+}
diff --git a/app/screenshot.css b/app/screenshot.css
index 51262d4..b892317 100644
--- a/app/screenshot.css
+++ b/app/screenshot.css
@@ -51,12 +51,19 @@
color: var(--shot-bar-text);
pointer-events: none;
}
-/* Never smaller than 80% of the capture (640 px of 800): the editor's 15 px
- text then shows at 12 px. A narrower screen scrolls the image sideways. */
+/* An editor shot is captured 800 px wide at font 14. Framed at most 720 px
+ wide, its text is never larger than 12.6 px, the size of the site's code
+ blocks, however wide the column grows. */
+.screenshot:not(.screenshot-platform) {
+ max-width: 720px;
+}
+/* Never smaller than 80% of the capture (640 px of 800): the editor's text
+ then shows at 11.2 px. A narrower screen scrolls the image sideways. */
.screenshot-scroll {
overflow-x: auto;
}
-.prose .screenshot img {
+.prose .screenshot img,
+.prose .screenshot video {
display: block;
width: 100%;
max-width: none;
@@ -64,11 +71,11 @@
height: auto;
margin: 0;
}
-.prose .screenshot img.screenshot-dark,
-.dark .prose .screenshot img.screenshot-light {
+.prose .screenshot .screenshot-dark,
+.dark .prose .screenshot .screenshot-light {
display: none;
}
-.dark .prose .screenshot img.screenshot-dark {
+.dark .prose .screenshot .screenshot-dark {
display: block;
}
.screenshot figcaption {
diff --git a/ci-templates/azure-devops/azure-pipelines.yml b/ci-templates/azure-devops/azure-pipelines.yml
index 5b58a29..6224615 100644
--- a/ci-templates/azure-devops/azure-pipelines.yml
+++ b/ci-templates/azure-devops/azure-pipelines.yml
@@ -33,7 +33,7 @@ stages:
version: 22.x
# An exact version: change it when you upgrade, as
# https://docs.ingestron.io/docs/how-to/upgrade#choose-the-version-you-run says.
- - script: npm install -g ingestron@2.4.0
+ - script: npm install -g ingestron@2.5.0
displayName: Install Ingestron
- script: ingestron check
displayName: ingestron check
diff --git a/ci-templates/azure-devops/ingestron-check.yml b/ci-templates/azure-devops/ingestron-check.yml
index ee04b68..9ece17d 100644
--- a/ci-templates/azure-devops/ingestron-check.yml
+++ b/ci-templates/azure-devops/ingestron-check.yml
@@ -15,7 +15,7 @@ steps:
version: 22.x
# An exact version: change it when you upgrade, as
# https://docs.ingestron.io/docs/how-to/upgrade#choose-the-version-you-run says.
- - script: npm install -g ingestron@2.4.0
+ - script: npm install -g ingestron@2.5.0
displayName: Install Ingestron
- script: ingestron check
displayName: ingestron check
diff --git a/ci-templates/github-actions/deploy.yml b/ci-templates/github-actions/deploy.yml
index bd2ae6c..2a48c7f 100644
--- a/ci-templates/github-actions/deploy.yml
+++ b/ci-templates/github-actions/deploy.yml
@@ -29,7 +29,7 @@ jobs:
- uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
with:
node-version: 22
- - run: npm install -g ingestron@2.4.0
+ - run: npm install -g ingestron@2.5.0
- run: ingestron check
build:
diff --git a/ci-templates/github-actions/ingestron-check.yml b/ci-templates/github-actions/ingestron-check.yml
index 47ba79f..b36e0c1 100644
--- a/ci-templates/github-actions/ingestron-check.yml
+++ b/ci-templates/github-actions/ingestron-check.yml
@@ -18,5 +18,5 @@ jobs:
- uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
with:
node-version: 22
- - run: npm install -g ingestron@2.4.0
+ - run: npm install -g ingestron@2.5.0
- run: ingestron check
diff --git a/components/connector-chooser.tsx b/components/connector-chooser.tsx
new file mode 100644
index 0000000..2679839
--- /dev/null
+++ b/components/connector-chooser.tsx
@@ -0,0 +1,29 @@
+// The page at a connector's old URL, /docs/connectors/, which the CLI and
+// the editor link to: each platform that loads the connector has its own page.
+import { PlatformChooserLinks } from "@/components/platform-choice";
+import { platforms } from "@/lib/platforms";
+import { connectorsFor } from "@/lib/reference";
+
+/**
+ * Links to each platform's page for a connector, or, without `id`, to each
+ * platform's list of connectors.
+ * @example
+ */
+export function ConnectorChooser({ id }: { id?: string }) {
+ const versions = platforms
+ .filter((platform) =>
+ id ? connectorsFor(platform.id).includes(id) : connectorsFor(platform.id).length > 0,
+ )
+ .map((platform) => platform.id);
+ if (versions.length === 0) throw new Error(`ConnectorChooser: no platform loads "${id}".`);
+ return (
+ <>
+
+ {id
+ ? "Each platform that loads this source has its own page. Choose yours:"
+ : "Each platform has its own list of the sources it loads. Choose yours:"}
+
+
+ >
+ );
+}
diff --git a/components/connector-picker.tsx b/components/connector-picker.tsx
new file mode 100644
index 0000000..efd4189
--- /dev/null
+++ b/components/connector-picker.tsx
@@ -0,0 +1,77 @@
+"use client";
+// At the top of a platform's Connectors sidebar, which lists categories rather
+// than every connector: type part of a source's name and choose it from the
+// connectors that platform loads. A search box and a list of links, so the
+// keyboard and screen readers use it as they use any links.
+import Link from "next/link";
+import { usePathname } from "next/navigation";
+import { useId, useState } from "react";
+
+/** One connector a platform loads. */
+export interface PickerEntry {
+ id: string;
+ title: string;
+}
+
+/**
+ * The connectors whose name has a word starting with the query, or whose id
+ * contains it; names that start with it first; at most ten.
+ * @example connectorMatches(list, "sql") // SQL Server, Azure SQL Database, MySQL
+ */
+export function connectorMatches(list: PickerEntry[], query: string): PickerEntry[] {
+ const q = query.trim().toLowerCase();
+ if (!q) return [];
+ const rank = (entry: PickerEntry) => {
+ const title = entry.title.toLowerCase();
+ if (title.startsWith(q)) return 0;
+ if (title.split(/[\s/()-]+/).some((word) => word.startsWith(q))) return 1;
+ if (entry.id.includes(q) || title.includes(q)) return 2;
+ return 3;
+ };
+ return list
+ .map((entry, index) => ({ entry, index, rank: rank(entry) }))
+ .filter((item) => item.rank < 3)
+ .sort((a, b) => a.rank - b.rank || a.index - b.index)
+ .slice(0, 10)
+ .map((item) => item.entry);
+}
+
+/** The picker, on a platform's Connectors pages only.
+ * @example
+ */
+export function ConnectorPicker({ lists }: { lists: Record }) {
+ const path = usePathname();
+ const [query, setQuery] = useState("");
+ const input = useId();
+ const platform = /^\/docs\/([a-z]+)\/connectors(?:\/|$)/.exec(path ?? "")?.[1];
+ const list = platform ? lists[platform] : undefined;
+ if (!platform || !list) return null;
+ const found = connectorMatches(list, query);
+ return (
+
+ );
+}
diff --git a/components/mdx.tsx b/components/mdx.tsx
index 8cf48d4..a261352 100644
--- a/components/mdx.tsx
+++ b/components/mdx.tsx
@@ -9,6 +9,8 @@ import { MarkdownTable } from "@/components/table-frame";
import { BuildTree, Example, ExampleDownload, Request, Template } from "@/components/example";
import { FlowDiagram } from "@/components/flow-diagram";
import { OldAnchor } from "@/components/old-anchor";
+import { ConnectorChooser } from "@/components/connector-chooser";
+import { PlatformChooser } from "@/components/platform-chooser";
import { Path, Paths } from "@/components/hub";
import { RouteComparison } from "@/components/reference/route-comparison";
import { RouteFacts } from "@/components/reference/route-facts";
@@ -38,7 +40,7 @@ import {
} from "@/components/reference/pack-intent";
import { KitFacts, KitFile, KitIntent, KitList } from "@/components/reference/kits";
import { RecipeCatalog, RecipeList, RecipePlaceholders } from "@/components/reference/recipes";
-import { ErrorCodes, LoadingCodes } from "@/components/reference/error-codes";
+import { ErrorCodes, LoadingCodes, OutputFolderProblems } from "@/components/reference/error-codes";
import { ExtensionCommands } from "@/components/reference/extension-commands";
import { ExamplesList } from "@/components/reference/examples-list";
import { PackMethods } from "@/components/reference/pack-methods";
@@ -52,7 +54,7 @@ import {
PackNames,
PlatformChecks,
} from "@/components/reference/versions";
-import { Screenshot } from "@/components/screenshot";
+import { Clip, Screenshot } from "@/components/screenshot";
import { Tool, ToolSteps } from "@/components/tool-steps";
export function getMDXComponents(components?: MDXComponents) {
@@ -72,12 +74,15 @@ export function getMDXComponents(components?: MDXComponents) {
table: MarkdownTable,
NamingVariables,
DocLinks,
+ PlatformChooser,
+ ConnectorChooser,
Example,
ExampleDownload,
BuildTree,
Request,
Template,
Screenshot,
+ Clip,
Tool,
ToolSteps,
SettingsTable,
@@ -93,6 +98,7 @@ export function getMDXComponents(components?: MDXComponents) {
DefaultLayers,
SamplesCatalogue,
ErrorCodes,
+ OutputFolderProblems,
LoadingCodes,
DocumentFields,
ContractFields,
diff --git a/components/platform-choice.tsx b/components/platform-choice.tsx
new file mode 100644
index 0000000..63f7c72
--- /dev/null
+++ b/components/platform-choice.tsx
@@ -0,0 +1,91 @@
+"use client";
+// The reader's platform: remembered from the last platform page they opened,
+// so an old guide link goes straight to their version, and shared pages can
+// offer the way back. Browser storage only; everything works without it.
+import Link from "next/link";
+import { usePathname } from "next/navigation";
+import { useEffect, useState } from "react";
+import { platformIds, platformOfPath, platforms, siteMark } from "@/lib/platforms";
+
+const key = "ingestron-platform";
+
+/** The remembered platform id, or undefined when there is none or storage is blocked. */
+export function rememberedPlatform(): string | undefined {
+ try {
+ const id = window.localStorage.getItem(key) ?? undefined;
+ return id && (platformIds as string[]).includes(id) ? id : undefined;
+ } catch {
+ return undefined;
+ }
+}
+
+/**
+ * Remembers a platform, and marks so the header shows its
+ * tabs (app/navigation.css). With storage blocked, only the mark is set.
+ */
+export function rememberPlatform(id: string): void {
+ document.documentElement.dataset.site = id;
+ try {
+ window.localStorage.setItem(key, id);
+ } catch {
+ // Storage blocked: nothing is remembered.
+ }
+}
+
+/**
+ * Remembers the platform of each platform page the reader opens, and on every
+ * page change, not only a full load, marks with the platform
+ * whose tabs to show. Renders nothing.
+ */
+export function PlatformMemory() {
+ const path = usePathname();
+ useEffect(() => {
+ const id = platformOfPath(path);
+ if (id) rememberPlatform(id);
+ const mark = siteMark(path, rememberedPlatform());
+ if (mark) document.documentElement.dataset.site = mark;
+ else delete document.documentElement.dataset.site;
+ }, [path]);
+ return null;
+}
+
+/**
+ * The versions of a page that moved into the platform sites, at its old URL.
+ * With a remembered platform that has the page, it goes there at once,
+ * keeping the link's #section.
+ * @example
+ */
+export function PlatformChooserLinks({ page, versions }: { page: string; versions: string[] }) {
+ const [hash, setHash] = useState("");
+ useEffect(() => {
+ setHash(window.location.hash);
+ const id = rememberedPlatform();
+ if (id && versions.includes(id))
+ window.location.replace(`/docs/${id}/${page}${window.location.hash}`);
+ }, [page, versions]);
+ return (
+
+ );
+}
+
+/** "Continue with Azure Synapse" on the home page, when a platform is remembered. */
+export function ContinueWithPlatform() {
+ const [id, setId] = useState();
+ useEffect(() => setId(rememberedPlatform()), []);
+ const platform = platforms.find((item) => item.id === id);
+ if (!platform) return null;
+ return (
+
+ Continue with {platform.title}
+
+ );
+}
diff --git a/components/platform-chooser.tsx b/components/platform-chooser.tsx
new file mode 100644
index 0000000..14b3941
--- /dev/null
+++ b/components/platform-chooser.tsx
@@ -0,0 +1,19 @@
+// The page at a shared page's old URL: the page is now one per platform.
+import { pageName, sharedPages } from "@/lib/platforms";
+import { PlatformChooserLinks } from "@/components/platform-choice";
+
+/**
+ * Links to each platform's version of a shared page.
+ * @example
+ */
+export function PlatformChooser({ page }: { page: string }) {
+ const shared = sharedPages[page];
+ if (!shared)
+ throw new Error(`PlatformChooser: "${page}" is not a shared page in lib/platforms.ts.`);
+ return (
+ <>
+
This page has a version for each platform. Choose yours:
+
+ >
+ );
+}
diff --git a/components/platform-picker.tsx b/components/platform-picker.tsx
new file mode 100644
index 0000000..2041e9b
--- /dev/null
+++ b/components/platform-picker.tsx
@@ -0,0 +1,178 @@
+"use client";
+// The site's main switch, in the header after the logo: the platform whose
+// docs you are reading. Choosing one opens the same page on it (or that
+// section's first page) and remembers it, so the product-wide pages keep
+// showing its tabs. A button and a list of choices: arrows move between
+// them, Enter chooses, Escape closes; with more than eight platforms a filter
+// box and provider groups appear. The combinations come last, after a line,
+// under "Combined" (lib/platforms.ts's pickerGroups).
+import { useRouter } from "next/navigation";
+import { usePathname } from "next/navigation";
+import { useEffect, useId, useRef, useState, type CSSProperties, type KeyboardEvent } from "react";
+import { createPortal } from "react-dom";
+import { rememberPlatform, rememberedPlatform } from "@/components/platform-choice";
+import {
+ manyPlatforms,
+ pickerGroups,
+ platformOfPath,
+ platforms,
+ singlePlatforms,
+ switchTarget,
+ type Platform,
+ type Sites,
+} from "@/lib/platforms";
+
+/** The picker.
+ * @example
+ */
+export function PlatformPicker({ sites }: { sites: Sites }) {
+ const path = usePathname() ?? "";
+ const router = useRouter();
+ const [chosen, setChosen] = useState(platformOfPath(path));
+ const [open, setOpen] = useState(false);
+ const [filter, setFilter] = useState("");
+ const list = useId();
+ const button = useRef(null);
+ const box = useRef(null);
+ const [place, setPlace] = useState();
+
+ // The page's platform, else the one chosen last.
+ useEffect(() => {
+ setChosen(platformOfPath(path) ?? (rememberedPlatform() as Platform["id"] | undefined));
+ }, [path]);
+
+ // The list is drawn over the page, outside the header, under the button and
+ // inside the window, so no sticky bar covers it and it never runs off a phone.
+ useEffect(() => {
+ if (!open || !button.current) return;
+ const at = () => {
+ const rect = button.current!.getBoundingClientRect();
+ const width = Math.min(352, window.innerWidth - 32);
+ setPlace({
+ top: rect.bottom + 6,
+ left: Math.max(16, Math.min(rect.left, window.innerWidth - width - 16)),
+ width,
+ // All the room below the button, so the combinations show too.
+ maxHeight: window.innerHeight - rect.bottom - 22,
+ });
+ };
+ at();
+ window.addEventListener("resize", at);
+ window.addEventListener("scroll", at, { passive: true });
+ return () => {
+ window.removeEventListener("resize", at);
+ window.removeEventListener("scroll", at);
+ };
+ }, [open]);
+
+ // Opening puts focus on the current choice; a click outside closes.
+ useEffect(() => {
+ if (!open || !place) return;
+ const items = box.current?.querySelectorAll("[data-choice]");
+ const current = box.current?.querySelector('[aria-current="true"]');
+ (box.current?.querySelector("input") ?? current ?? items?.[0])?.focus();
+ const away = (event: PointerEvent) => {
+ if (!box.current?.contains(event.target as Node) && event.target !== button.current)
+ setOpen(false);
+ };
+ document.addEventListener("pointerdown", away);
+ return () => document.removeEventListener("pointerdown", away);
+ }, [open, place]);
+
+ const choose = (id: Platform["id"]) => {
+ rememberPlatform(id);
+ setChosen(id);
+ setOpen(false);
+ const target = switchTarget(`${path}${window.location.hash}`, id, sites);
+ if (target !== `${path}${window.location.hash}`) router.push(target);
+ button.current?.focus();
+ };
+
+ const onKey = (event: KeyboardEvent) => {
+ if (event.key === "Escape") {
+ // Close the list only: the menu or search behind it stays open.
+ event.stopPropagation();
+ event.preventDefault();
+ setOpen(false);
+ button.current?.focus();
+ return;
+ }
+ if (event.key !== "ArrowDown" && event.key !== "ArrowUp") return;
+ event.preventDefault();
+ const items = [...(box.current?.querySelectorAll("[data-choice]") ?? [])];
+ const at = items.indexOf(document.activeElement as HTMLElement);
+ const next = event.key === "ArrowDown" ? at + 1 : at - 1;
+ items[(next + items.length) % items.length]?.focus();
+ };
+
+ const current = platforms.find((platform) => platform.id === chosen);
+ const many = singlePlatforms.length > manyPlatforms;
+ const groups = pickerGroups(filter);
+
+ return (
+
+
+ {open &&
+ place &&
+ createPortal(
+
{
+ const next = event.relatedTarget as Node | null;
+ if (next && !box.current?.contains(next) && next !== button.current) setOpen(false);
+ }}
+ >
+ {many && (
+ setFilter(event.target.value)}
+ />
+ )}
+ {groups.map((group) => (
+
+ );
+}
diff --git a/components/platform-search.tsx b/components/platform-search.tsx
new file mode 100644
index 0000000..42c19c3
--- /dev/null
+++ b/components/platform-search.tsx
@@ -0,0 +1,57 @@
+"use client";
+// The search dialog, limited to one platform's path and the shared sections:
+// the platform of the page you are on, else the one you last read, else all.
+// Clear the filter to search every platform. Built from Fumadocs' dialog
+// parts, with the platform filter inside the dialog.
+import {
+ SearchDialog,
+ SearchDialogClose,
+ SearchDialogContent,
+ SearchDialogFooter,
+ SearchDialogHeader,
+ SearchDialogIcon,
+ SearchDialogInput,
+ SearchDialogList,
+ SearchDialogOverlay,
+ TagsList,
+ TagsListItem,
+ type SharedProps,
+} from "fumadocs-ui/components/dialog/search";
+import { useDocsSearch } from "fumadocs-core/search/client";
+import { staticClient } from "fumadocs-core/search/client/orama-static";
+import { usePathname } from "next/navigation";
+import { useEffect, useState } from "react";
+import { platformOfPath, platforms } from "@/lib/platforms";
+import { rememberedPlatform } from "@/components/platform-choice";
+
+export function PlatformSearchDialog(props: SharedProps) {
+ const path = usePathname();
+ const [tag, setTag] = useState(platformOfPath(path));
+ // Each time the dialog opens, the filter starts at the reader's platform.
+ useEffect(() => {
+ if (props.open) setTag(platformOfPath(path) ?? rememberedPlatform());
+ }, [props.open, path]);
+ const search = useDocsSearch({ client: staticClient({ tag }) });
+ return (
+
+
+
+
+
+
+
+
+
+
+
+ {platforms.map((platform) => (
+
+ {platform.title}
+
+ ))}
+
+
+
+
+ );
+}
diff --git a/components/reference/connector-facts.tsx b/components/reference/connector-facts.tsx
index 79bc78c..377eceb 100644
--- a/components/reference/connector-facts.tsx
+++ b/components/reference/connector-facts.tsx
@@ -3,32 +3,55 @@ import { Fragment, type ReactNode } from "react";
import { projectRoutes } from "@ingestron/core";
import { ingestionRows } from "@/lib/connector-settings";
import { listExamples } from "@/lib/examples";
-import { connectorEntries, packName, type ConnectorEntry } from "@/lib/reference";
+import { platformById, type Platform } from "@/lib/platforms";
+import {
+ connectorEntries,
+ packName,
+ platformUsedPacks,
+ type ConnectorEntry,
+} from "@/lib/reference";
import { connectorCommand, starter, starterCommand } from "@/lib/routes";
-/** A compact summary of each pack that reads the connector, the routes it starts and the command for each, then the starters that read it.
- * @example
+/** A compact summary of each pack that reads the connector, the routes it starts and the command for each, then the starters that read it; with `platform`, only that platform's.
+ * @example
*/
-export function ConnectorFacts({ id }: { id: string }) {
- const entries = connectorEntries(id);
+export function ConnectorFacts({ id, platform }: { id: string; platform?: Platform["id"] }) {
+ const entries = connectorEntries(id).filter((entry) => routesOf(id, entry, platform).length > 0);
if (entries.length === 0) throw new Error(`No pack reads connector "${id}".`);
return (
@@ -46,11 +69,13 @@ function Route({ id, entry }: { id: string; entry: ConnectorEntry }) {
function routeFacts(
id: string,
- { pack, connector, connectivity }: ConnectorEntry,
+ entry: ConnectorEntry,
+ platform?: Platform["id"],
): [string, ReactNode][] {
+ const { pack, connector, connectivity } = entry;
const { reading, maturity } = connector;
const modes = ingestionRows(id, pack).find((field) => field.path.join(".") === "mode")?.values;
- const routes = projectRoutes(id).filter((route) => route.method === connector.method);
+ const routes = routesOf(id, entry, platform);
if (routes.length === 0) throw new Error(`No route reads "${id}" with ${connector.method}.`);
const facts: [string, ReactNode][] = [
[
@@ -88,10 +113,12 @@ function routeFacts(
return facts;
}
-/** The built-in starters whose connections use the connector, the one with no cloud first. */
-function Starters({ id }: { id: string }) {
+/** The built-in starters whose connections use the connector, the one with no cloud first; on a platform, those its routes build. */
+function Starters({ id, platform }: { id: string; platform?: Platform["id"] }) {
+ const used = platform && platformUsedPacks(platform);
const starters = listExamples()
.filter((example) => example.template && example.facts.connectors.includes(id))
+ .filter((example) => !used || example.facts.packs.every((pack) => used.has(pack)))
.sort((a, b) => Number(b.template === starter) - Number(a.template === starter));
if (starters.length === 0) return null;
return (
diff --git a/components/reference/connector-page.tsx b/components/reference/connector-page.tsx
index 1f52979..44b24c9 100644
--- a/components/reference/connector-page.tsx
+++ b/components/reference/connector-page.tsx
@@ -15,6 +15,7 @@ import {
type Part,
type Section,
} from "@/lib/connector-page";
+import type { Platform } from "@/lib/platforms";
import { ConnectorFacts } from "./connector-facts";
import { ConnectorSettings } from "./connector-settings";
import { ExamplesList } from "./examples-list";
@@ -30,10 +31,19 @@ export function ConnectorNote({ children }: { at: string; children?: ReactNode }
return <>{children}>;
}
-/** One connector's page: its routes, how to connect and read with each pack, and examples.
- * @example
+/** One connector's page: its routes, how to connect and read with each pack, and examples;
+ * with `platform`, that platform's copy, with only its routes and packs.
+ * @example
*/
-export function ConnectorPage({ id, children }: { id: string; children?: ReactNode }) {
+export function ConnectorPage({
+ id,
+ platform,
+ children,
+}: {
+ id: string;
+ platform?: Platform["id"];
+ children?: ReactNode;
+}) {
const notes = Children.toArray(children).map((child) => {
const at = isValidElement<{ at?: string }>(child) ? child.props.at : undefined;
if (!at)
@@ -45,7 +55,7 @@ export function ConnectorPage({ id, children }: { id: string; children?: ReactNo
if (unknown.length > 0)
throw new Error(`connectors/${id}: no slot ${unknown.join(", ")} on this page.`);
const slot = (name: string) => notes.filter((note) => note.at === name).map((note) => note.child);
- const sections = connectorSections(id);
+ const sections = connectorSections(id, platform);
const first = sections.find((section) => section.entry)!;
return (
<>
@@ -55,9 +65,9 @@ export function ConnectorPage({ id, children }: { id: string; children?: ReactNo
{section.title}
- {section.id === "routes" && }
+ {section.id === "routes" && }
{section.id === "source-metadata" && }
- {section.id === "examples" && }
+ {section.id === "examples" && }
{section.items.map((item) =>
"slot" in item ? (
{slot(item.slot)}
@@ -146,8 +156,8 @@ function Shown({
case "watermarks":
return (
<>
-
-
+
+
>
);
case "settings":
diff --git a/components/reference/contract-types.tsx b/components/reference/contract-types.tsx
index abd6e96..d8b0ed8 100644
--- a/components/reference/contract-types.tsx
+++ b/components/reference/contract-types.tsx
@@ -1,5 +1,6 @@
// The column type each pack gives a contract's logicalType, from the packs' own defaults.
-import { packName, packs } from "@/lib/reference";
+import type { Platform } from "@/lib/platforms";
+import { packName, packs, platformUsedPacks } from "@/lib/reference";
import { TableFrame } from "@/components/table-frame";
/** The logical types of ODCS v3.1.0, in the order the contract notes explain them. */
@@ -15,9 +16,16 @@ const logicalTypes = [
"array",
];
-/** One row per logical type, one column per pack that types tables. */
-export function ContractTypes() {
- const typed = packs.filter((pack) => Object.keys(pack.defaults.types ?? {}).length > 0);
+/** One row per logical type, one column per pack that types tables; with
+ * `platform`, only the packs that platform's routes use.
+ * @example
+ */
+export function ContractTypes({ platform }: { platform?: Platform["id"] }) {
+ const used = platform && platformUsedPacks(platform);
+ const typed = packs.filter(
+ (pack) => Object.keys(pack.defaults.types ?? {}).length > 0 && (!used || used.has(pack.id)),
+ );
+ if (typed.length === 0) throw new Error(`No pack types tables on ${platform}.`);
return (
diff --git a/components/reference/error-codes.tsx b/components/reference/error-codes.tsx
index df786ab..103d1c3 100644
--- a/components/reference/error-codes.tsx
+++ b/components/reference/error-codes.tsx
@@ -1,6 +1,7 @@
// Every code Ingestron reports, with its meaning, the levels it is reported
// at and its fix from the code, each with an anchor that problems in the
// editor link to.
+import { problemHelp } from "@ingestron/core";
import { codeHelp, loadingCodeHelp } from "@ingestron/definitions";
import { Inline } from "./inline";
@@ -9,6 +10,11 @@ export function ErrorCodes() {
return ;
}
+/** The problems with an output folder, each with the command that reports it. */
+export function OutputFolderProblems() {
+ return ;
+}
+
/** The codes found while loading files, grouped by when they are found. */
export function LoadingCodes() {
const entries = Object.entries(loadingCodeHelp);
@@ -37,7 +43,10 @@ function Codes({
entries,
level: Heading,
}: {
- entries: [string, { meaning: string; fix: string; severities: readonly string[] }][];
+ entries: [
+ string,
+ { meaning: string; fix: string; severities?: readonly string[]; reportedBy?: string },
+ ][];
level: "h3" | "h4";
}) {
return (
@@ -46,12 +55,17 @@ function Codes({
{code}
- {help.severities.length === 0 && " (removed)"}
+ {help.severities?.length === 0 && " (removed)"}
diff --git a/components/reference/pack-defaults.tsx b/components/reference/pack-defaults.tsx
index 5e66eaf..1f6a6f7 100644
--- a/components/reference/pack-defaults.tsx
+++ b/components/reference/pack-defaults.tsx
@@ -1,6 +1,7 @@
// The names and layers each pack gives a project whose standard sets none, from the packs' own defaults.
import type { Naming } from "@ingestron/definitions";
-import { packName, packs } from "@/lib/reference";
+import type { Platform } from "@/lib/platforms";
+import { packName, packs, platformUsedPacks } from "@/lib/reference";
import { TableFrame } from "@/components/table-frame";
/** The patterns a standard's `naming` can set, in the order the document reference lists them. */
@@ -11,12 +12,15 @@ const names: (keyof Naming)[] = ["catalog", "schema", "table", "pipeline", "job"
* factory's resources with its methods' `resource_names` instead, so its
* naming defaults name nothing it generates.
*/
-const naming = () => packs.filter((pack) => pack.id !== "adf");
+const allNaming = () => packs.filter((pack) => pack.id !== "adf");
/** One row per name, one column per pack that names tables.
* @example
*/
-export function PackNaming() {
+/** With `platform`, only the packs that platform's routes use. */
+export function PackNaming({ platform }: { platform?: Platform["id"] } = {}) {
+ const used = platform && platformUsedPacks(platform);
+ const naming = () => allNaming().filter((pack) => !used || used.has(pack.id));
return (
diff --git a/components/reference/project-sources.tsx b/components/reference/project-sources.tsx
index 8f37689..268ecf9 100644
--- a/components/reference/project-sources.tsx
+++ b/components/reference/project-sources.tsx
@@ -3,30 +3,47 @@
import { projectSources, projectRoutes } from "@ingestron/core";
import { Heading } from "fumadocs-ui/components/heading";
import { categoryAnchor, sourceCategories } from "@/lib/connector-page";
+import { platformById, type Platform } from "@/lib/platforms";
-/** Every source, under a heading per category, with the routes it can take.
- * @example
+/** Every source a platform loads, under a heading per category, with the routes it can take there.
+ * @example
*/
-export function ProjectSources() {
- const sources = projectSources();
- return sourceCategories().map((name) => (
-
{deploy ? (
<>
diff --git a/components/reference/vscode-commands.json b/components/reference/vscode-commands.json
index 5e02f39..3cdd2d7 100644
--- a/components/reference/vscode-commands.json
+++ b/components/reference/vscode-commands.json
@@ -73,5 +73,20 @@
"command": "ingestron.addAssistantFiles",
"title": "Add assistant files",
"category": "Ingestron"
+ },
+ {
+ "command": "ingestron.addConnection",
+ "title": "Add connection",
+ "category": "Ingestron"
+ },
+ {
+ "command": "ingestron.addFlow",
+ "title": "Add flow",
+ "category": "Ingestron"
+ },
+ {
+ "command": "ingestron.draftContractFromMetadata",
+ "title": "Draft contract from metadata",
+ "category": "Ingestron"
}
]
diff --git a/components/reference/vscode-extension.json b/components/reference/vscode-extension.json
index d9f6f21..1651884 100644
--- a/components/reference/vscode-extension.json
+++ b/components/reference/vscode-extension.json
@@ -1,3 +1,3 @@
{
- "version": "2.3.0"
+ "version": "2.4.0"
}
diff --git a/components/screenshot.tsx b/components/screenshot.tsx
index 772d048..badef83 100644
--- a/components/screenshot.tsx
+++ b/components/screenshot.tsx
@@ -8,9 +8,11 @@
// with the `label` it needs and the date it was `checked` against the
// platform, which ends the caption. A platform shot is cropped to its subject
// and shown at its own size, never larger, in a frame no wider than the
-// image, and links to the image to open it full size.
-import { readFileSync } from "node:fs";
+// image, and links to the image to open it full size. A short recording of
+// the editor is a `Clip`, in the same frame.
+import { existsSync, readFileSync } from "node:fs";
import { join } from "node:path";
+import type { ReactNode } from "react";
import { ScreenshotScroll } from "@/components/screenshot-scroll";
/** A PNG's size in pixels, from its header: the build fails if the file is missing. */
@@ -117,6 +119,90 @@ export function Screenshot({
decoding="async"
/>
));
+ return (
+
+ {platform ? (
+
+ {shown}
+
+ ) : (
+ shown
+ )}
+
+ );
+}
+
+/**
+ * A short screen recording in the editor's frame: a `light` and a `dark` MP4,
+ * each with a PNG of its last frame beside it, of the same name, as the
+ * poster. It is silent and plays only when the reader starts it, so nothing
+ * moves on the page by itself, and it loads nothing until then.
+ * @example
+ */
+export function Clip({
+ light,
+ dark,
+ caption,
+ alt,
+}: {
+ /** The light theme's MP4; its poster is the PNG of the same name. */
+ light: string;
+ /** The dark theme's MP4, the same size as the light one. */
+ dark: string;
+ /** Says what the clip shows. */
+ caption: string;
+ /** Describes the clip, step by step, for a reader who cannot see it; the caption when left out. */
+ alt?: string;
+}) {
+ const clips = [
+ { src: light, className: "screenshot-light" },
+ { src: dark, className: "screenshot-dark" },
+ ].map((clip) => ({ ...clip, poster: clip.src.replace(/\.mp4$/, ".png") }));
+ for (const clip of clips) {
+ if (!existsSync(join(process.cwd(), "public", clip.src))) {
+ throw new Error(`Clip: no such file ${clip.src}`);
+ }
+ }
+ const [size, other] = clips.map((clip) => pngSize(clip.poster));
+ if (size.width !== other.width || size.height !== other.height) {
+ throw new Error(`Clip: ${light} and ${dark} differ in size`);
+ }
+ const name = alt ?? caption;
+ return (
+
+ {clips.map((clip) => (
+
+ ))}
+
+ );
+}
+
+/** The window round a screenshot or a clip: the bar, the scrolling frame and the caption. */
+function Frame({
+ platform,
+ bar,
+ name,
+ text,
+ children,
+}: {
+ platform: boolean;
+ bar: string;
+ name: string;
+ text: string;
+ children: ReactNode;
+}) {
return (
@@ -129,21 +215,10 @@ export function Screenshot({
{bar}
{/* On a narrow screen the image keeps a readable size and scrolls in a
- frame named after the shot, which takes focus only while it scrolls. */}
-
- {/* With two captures, the page shows one of them with CSS, as it does the logo. */}
- {platform ? (
-
- {shown}
-
- ) : (
- shown
- )}
-
+ frame named after the shot, which takes focus only while it scrolls.
+ With two captures, the page shows one of them with CSS, as it does
+ the logo. */}
+ {children}{text}
diff --git a/components/section-companion.tsx b/components/section-companion.tsx
new file mode 100644
index 0000000..df87238
--- /dev/null
+++ b/components/section-companion.tsx
@@ -0,0 +1,60 @@
+"use client";
+// Above a Concepts or Reference sidebar: the other half of that section. A
+// platform's own page (its troubleshooting, what it cannot apply) lists the
+// pages every platform shares; a shared page lists the remembered platform's
+// own. Links only; the sidebar below is the section's own tree.
+import Link from "next/link";
+import { usePathname } from "next/navigation";
+import { useEffect, useState } from "react";
+import { rememberedPlatform } from "@/components/platform-choice";
+import type { CompanionData, CompanionLink } from "@/lib/areas";
+import { platformById, platformOfPath, type Platform } from "@/lib/platforms";
+
+/**
+ * The group for a page: its heading and links, or undefined outside Concepts
+ * and Reference, or on a shared page with no platform remembered.
+ * @example companionFor("/docs/reference/cli", "duckdb", data) // { heading: "On DuckDB", links: […] }
+ */
+export function companionFor(
+ path: string,
+ remembered: string | undefined,
+ data: CompanionData,
+): { heading: string; links: CompanionLink[] } | undefined {
+ const page = path.length > 1 ? path.replace(/\/$/, "") : path;
+ const own = platformOfPath(page);
+ for (const section of Object.values(data)) {
+ if (own && section.platforms[own]?.some((link) => link.url === page))
+ return { heading: "For every platform", links: section.wide };
+ if (!own && remembered && section.wide.some((link) => link.url === page)) {
+ const links = section.platforms[remembered] ?? [];
+ if (links.length === 0) return undefined;
+ return { heading: `On ${platformById(remembered).title}`, links };
+ }
+ }
+ return undefined;
+}
+
+/** The group, in the sidebar's banner.
+ * @example
+ */
+export function SectionCompanion({ data }: { data: CompanionData }) {
+ const path = usePathname() ?? "";
+ const [remembered, setRemembered] = useState();
+ useEffect(() => setRemembered(rememberedPlatform() as Platform["id"] | undefined), [path]);
+ const found = companionFor(path, remembered, data);
+ if (!found) return null;
+ return (
+
+ );
+}
diff --git a/content/docs/(moved)/concepts/environments-and-platform.mdx b/content/docs/(moved)/concepts/environments-and-platform.mdx
new file mode 100644
index 0000000..e2cdf76
--- /dev/null
+++ b/content/docs/(moved)/concepts/environments-and-platform.mdx
@@ -0,0 +1,6 @@
+---
+title: Environments and the platform document
+description: Reuse one project's definitions with different values and shared infrastructure settings.
+---
+
+
diff --git a/content/docs/(moved)/concepts/not-applied.mdx b/content/docs/(moved)/concepts/not-applied.mdx
new file mode 100644
index 0000000..436f375
--- /dev/null
+++ b/content/docs/(moved)/concepts/not-applied.mdx
@@ -0,0 +1,6 @@
+---
+title: Not applied here
+description: What it means when the engine that keeps a table cannot apply something your definitions ask for, where Ingestron shows it, and what to do about it.
+---
+
+
diff --git a/content/docs/(moved)/connectors/adls-gen2.mdx b/content/docs/(moved)/connectors/adls-gen2.mdx
new file mode 100644
index 0000000..bc2a045
--- /dev/null
+++ b/content/docs/(moved)/connectors/adls-gen2.mdx
@@ -0,0 +1,6 @@
+---
+title: Azure Data Lake Storage Gen2
+description: "How Azure Data Lake Storage Gen2 is read: each platform that loads it has its own page."
+---
+
+
diff --git a/content/docs/(moved)/connectors/amazon-rds-oracle.mdx b/content/docs/(moved)/connectors/amazon-rds-oracle.mdx
new file mode 100644
index 0000000..9703041
--- /dev/null
+++ b/content/docs/(moved)/connectors/amazon-rds-oracle.mdx
@@ -0,0 +1,6 @@
+---
+title: Amazon RDS for Oracle
+description: "How Amazon RDS for Oracle is read: each platform that loads it has its own page."
+---
+
+
diff --git a/content/docs/(moved)/connectors/amazon-rds-sqlserver.mdx b/content/docs/(moved)/connectors/amazon-rds-sqlserver.mdx
new file mode 100644
index 0000000..103e232
--- /dev/null
+++ b/content/docs/(moved)/connectors/amazon-rds-sqlserver.mdx
@@ -0,0 +1,6 @@
+---
+title: Amazon RDS for SQL Server
+description: "How Amazon RDS for SQL Server is read: each platform that loads it has its own page."
+---
+
+
diff --git a/content/docs/(moved)/connectors/amazon-redshift.mdx b/content/docs/(moved)/connectors/amazon-redshift.mdx
new file mode 100644
index 0000000..ddde471
--- /dev/null
+++ b/content/docs/(moved)/connectors/amazon-redshift.mdx
@@ -0,0 +1,6 @@
+---
+title: Amazon Redshift
+description: "How Amazon Redshift is read: each platform that loads it has its own page."
+---
+
+
diff --git a/content/docs/(moved)/connectors/amazon-s3.mdx b/content/docs/(moved)/connectors/amazon-s3.mdx
new file mode 100644
index 0000000..a410e81
--- /dev/null
+++ b/content/docs/(moved)/connectors/amazon-s3.mdx
@@ -0,0 +1,6 @@
+---
+title: Amazon S3
+description: "How Amazon S3 is read: each platform that loads it has its own page."
+---
+
+
diff --git a/content/docs/(moved)/connectors/azure-blob.mdx b/content/docs/(moved)/connectors/azure-blob.mdx
new file mode 100644
index 0000000..b3379ba
--- /dev/null
+++ b/content/docs/(moved)/connectors/azure-blob.mdx
@@ -0,0 +1,6 @@
+---
+title: Azure Blob Storage
+description: "How Azure Blob Storage is read: each platform that loads it has its own page."
+---
+
+
diff --git a/content/docs/(moved)/connectors/azure-cosmos-db-mongodb.mdx b/content/docs/(moved)/connectors/azure-cosmos-db-mongodb.mdx
new file mode 100644
index 0000000..757992d
--- /dev/null
+++ b/content/docs/(moved)/connectors/azure-cosmos-db-mongodb.mdx
@@ -0,0 +1,6 @@
+---
+title: Azure Cosmos DB for MongoDB
+description: "How Azure Cosmos DB for MongoDB is read: each platform that loads it has its own page."
+---
+
+
diff --git a/content/docs/(moved)/connectors/azure-cosmos-db-nosql.mdx b/content/docs/(moved)/connectors/azure-cosmos-db-nosql.mdx
new file mode 100644
index 0000000..265cc18
--- /dev/null
+++ b/content/docs/(moved)/connectors/azure-cosmos-db-nosql.mdx
@@ -0,0 +1,6 @@
+---
+title: Azure Cosmos DB for NoSQL
+description: "How Azure Cosmos DB for NoSQL is read: each platform that loads it has its own page."
+---
+
+
diff --git a/content/docs/(moved)/connectors/azure-cosmos-postgresql.mdx b/content/docs/(moved)/connectors/azure-cosmos-postgresql.mdx
new file mode 100644
index 0000000..e19785e
--- /dev/null
+++ b/content/docs/(moved)/connectors/azure-cosmos-postgresql.mdx
@@ -0,0 +1,6 @@
+---
+title: Azure Cosmos DB for PostgreSQL
+description: "How Azure Cosmos DB for PostgreSQL is read: each platform that loads it has its own page."
+---
+
+
diff --git a/content/docs/(moved)/connectors/azure-data-explorer.mdx b/content/docs/(moved)/connectors/azure-data-explorer.mdx
new file mode 100644
index 0000000..e46c20b
--- /dev/null
+++ b/content/docs/(moved)/connectors/azure-data-explorer.mdx
@@ -0,0 +1,6 @@
+---
+title: Azure Data Explorer
+description: "How Azure Data Explorer is read: each platform that loads it has its own page."
+---
+
+
diff --git a/content/docs/(moved)/connectors/azure-databricks-delta.mdx b/content/docs/(moved)/connectors/azure-databricks-delta.mdx
new file mode 100644
index 0000000..c6a3b6f
--- /dev/null
+++ b/content/docs/(moved)/connectors/azure-databricks-delta.mdx
@@ -0,0 +1,6 @@
+---
+title: Azure Databricks Delta Lake
+description: "How Azure Databricks Delta Lake is read: each platform that loads it has its own page."
+---
+
+
diff --git a/content/docs/(moved)/connectors/azure-files.mdx b/content/docs/(moved)/connectors/azure-files.mdx
new file mode 100644
index 0000000..f164bae
--- /dev/null
+++ b/content/docs/(moved)/connectors/azure-files.mdx
@@ -0,0 +1,6 @@
+---
+title: Azure Files
+description: "How Azure Files is read: each platform that loads it has its own page."
+---
+
+
diff --git a/content/docs/(moved)/connectors/azure-mysql.mdx b/content/docs/(moved)/connectors/azure-mysql.mdx
new file mode 100644
index 0000000..8a3b380
--- /dev/null
+++ b/content/docs/(moved)/connectors/azure-mysql.mdx
@@ -0,0 +1,6 @@
+---
+title: Azure Database for MySQL
+description: "How Azure Database for MySQL is read: each platform that loads it has its own page."
+---
+
+
diff --git a/content/docs/(moved)/connectors/azure-postgresql.mdx b/content/docs/(moved)/connectors/azure-postgresql.mdx
new file mode 100644
index 0000000..825317a
--- /dev/null
+++ b/content/docs/(moved)/connectors/azure-postgresql.mdx
@@ -0,0 +1,6 @@
+---
+title: Azure Database for PostgreSQL
+description: "How Azure Database for PostgreSQL is read: each platform that loads it has its own page."
+---
+
+
diff --git a/content/docs/(moved)/connectors/azure-sql-database.mdx b/content/docs/(moved)/connectors/azure-sql-database.mdx
new file mode 100644
index 0000000..d2cefa2
--- /dev/null
+++ b/content/docs/(moved)/connectors/azure-sql-database.mdx
@@ -0,0 +1,6 @@
+---
+title: Azure SQL Database
+description: "How Azure SQL Database is read: each platform that loads it has its own page."
+---
+
+
diff --git a/content/docs/(moved)/connectors/azure-sql-managed-instance.mdx b/content/docs/(moved)/connectors/azure-sql-managed-instance.mdx
new file mode 100644
index 0000000..86f6010
--- /dev/null
+++ b/content/docs/(moved)/connectors/azure-sql-managed-instance.mdx
@@ -0,0 +1,6 @@
+---
+title: Azure SQL Managed Instance
+description: "How Azure SQL Managed Instance is read: each platform that loads it has its own page."
+---
+
+
diff --git a/content/docs/(moved)/connectors/azure-synapse.mdx b/content/docs/(moved)/connectors/azure-synapse.mdx
new file mode 100644
index 0000000..ff4e75a
--- /dev/null
+++ b/content/docs/(moved)/connectors/azure-synapse.mdx
@@ -0,0 +1,6 @@
+---
+title: Azure Synapse Analytics
+description: "How Azure Synapse Analytics is read: each platform that loads it has its own page."
+---
+
+
diff --git a/content/docs/(moved)/connectors/azure-table-storage.mdx b/content/docs/(moved)/connectors/azure-table-storage.mdx
new file mode 100644
index 0000000..57811da
--- /dev/null
+++ b/content/docs/(moved)/connectors/azure-table-storage.mdx
@@ -0,0 +1,6 @@
+---
+title: Azure Table Storage
+description: "How Azure Table Storage is read: each platform that loads it has its own page."
+---
+
+
diff --git a/content/docs/(moved)/connectors/cassandra.mdx b/content/docs/(moved)/connectors/cassandra.mdx
new file mode 100644
index 0000000..aa61eda
--- /dev/null
+++ b/content/docs/(moved)/connectors/cassandra.mdx
@@ -0,0 +1,6 @@
+---
+title: Cassandra
+description: "How Cassandra is read: each platform that loads it has its own page."
+---
+
+
diff --git a/content/docs/(moved)/connectors/dataverse.mdx b/content/docs/(moved)/connectors/dataverse.mdx
new file mode 100644
index 0000000..1ea6a64
--- /dev/null
+++ b/content/docs/(moved)/connectors/dataverse.mdx
@@ -0,0 +1,6 @@
+---
+title: Dataverse
+description: "How Dataverse is read: each platform that loads it has its own page."
+---
+
+
diff --git a/content/docs/(moved)/connectors/db2.mdx b/content/docs/(moved)/connectors/db2.mdx
new file mode 100644
index 0000000..cc216d6
--- /dev/null
+++ b/content/docs/(moved)/connectors/db2.mdx
@@ -0,0 +1,6 @@
+---
+title: DB2
+description: "How DB2 is read: each platform that loads it has its own page."
+---
+
+
diff --git a/content/docs/(moved)/connectors/dynamics-365.mdx b/content/docs/(moved)/connectors/dynamics-365.mdx
new file mode 100644
index 0000000..bf8d6a7
--- /dev/null
+++ b/content/docs/(moved)/connectors/dynamics-365.mdx
@@ -0,0 +1,6 @@
+---
+title: Dynamics 365
+description: "How Dynamics 365 is read: each platform that loads it has its own page."
+---
+
+
diff --git a/content/docs/(moved)/connectors/dynamics-ax.mdx b/content/docs/(moved)/connectors/dynamics-ax.mdx
new file mode 100644
index 0000000..11d1256
--- /dev/null
+++ b/content/docs/(moved)/connectors/dynamics-ax.mdx
@@ -0,0 +1,6 @@
+---
+title: Dynamics AX
+description: "How Dynamics AX is read: each platform that loads it has its own page."
+---
+
+
diff --git a/content/docs/(moved)/connectors/dynamics-crm.mdx b/content/docs/(moved)/connectors/dynamics-crm.mdx
new file mode 100644
index 0000000..e85090a
--- /dev/null
+++ b/content/docs/(moved)/connectors/dynamics-crm.mdx
@@ -0,0 +1,6 @@
+---
+title: Dynamics CRM
+description: "How Dynamics CRM is read: each platform that loads it has its own page."
+---
+
+
diff --git a/content/docs/(moved)/connectors/fabric-lakehouse.mdx b/content/docs/(moved)/connectors/fabric-lakehouse.mdx
new file mode 100644
index 0000000..9835439
--- /dev/null
+++ b/content/docs/(moved)/connectors/fabric-lakehouse.mdx
@@ -0,0 +1,6 @@
+---
+title: Microsoft Fabric Lakehouse
+description: "How Microsoft Fabric Lakehouse is read: each platform that loads it has its own page."
+---
+
+
diff --git a/content/docs/(moved)/connectors/fabric-warehouse.mdx b/content/docs/(moved)/connectors/fabric-warehouse.mdx
new file mode 100644
index 0000000..c0b1bd1
--- /dev/null
+++ b/content/docs/(moved)/connectors/fabric-warehouse.mdx
@@ -0,0 +1,6 @@
+---
+title: Microsoft Fabric Warehouse
+description: "How Microsoft Fabric Warehouse is read: each platform that loads it has its own page."
+---
+
+
diff --git a/content/docs/(moved)/connectors/files.mdx b/content/docs/(moved)/connectors/files.mdx
new file mode 100644
index 0000000..bba034a
--- /dev/null
+++ b/content/docs/(moved)/connectors/files.mdx
@@ -0,0 +1,6 @@
+---
+title: Files
+description: "How Files is read: each platform that loads it has its own page."
+---
+
+
diff --git a/content/docs/(moved)/connectors/ftp.mdx b/content/docs/(moved)/connectors/ftp.mdx
new file mode 100644
index 0000000..aa0f2d3
--- /dev/null
+++ b/content/docs/(moved)/connectors/ftp.mdx
@@ -0,0 +1,6 @@
+---
+title: FTP
+description: "How FTP is read: each platform that loads it has its own page."
+---
+
+
diff --git a/content/docs/(moved)/connectors/google-ads.mdx b/content/docs/(moved)/connectors/google-ads.mdx
new file mode 100644
index 0000000..51edf12
--- /dev/null
+++ b/content/docs/(moved)/connectors/google-ads.mdx
@@ -0,0 +1,6 @@
+---
+title: Google Ads
+description: "How Google Ads is read: each platform that loads it has its own page."
+---
+
+
diff --git a/content/docs/(moved)/connectors/google-bigquery.mdx b/content/docs/(moved)/connectors/google-bigquery.mdx
new file mode 100644
index 0000000..3fa2683
--- /dev/null
+++ b/content/docs/(moved)/connectors/google-bigquery.mdx
@@ -0,0 +1,6 @@
+---
+title: Google BigQuery
+description: "How Google BigQuery is read: each platform that loads it has its own page."
+---
+
+
diff --git a/content/docs/(moved)/connectors/google-cloud-storage.mdx b/content/docs/(moved)/connectors/google-cloud-storage.mdx
new file mode 100644
index 0000000..02ef2b5
--- /dev/null
+++ b/content/docs/(moved)/connectors/google-cloud-storage.mdx
@@ -0,0 +1,6 @@
+---
+title: Google Cloud Storage
+description: "How Google Cloud Storage is read: each platform that loads it has its own page."
+---
+
+
diff --git a/content/docs/(moved)/connectors/greenplum.mdx b/content/docs/(moved)/connectors/greenplum.mdx
new file mode 100644
index 0000000..51b54e0
--- /dev/null
+++ b/content/docs/(moved)/connectors/greenplum.mdx
@@ -0,0 +1,6 @@
+---
+title: Greenplum
+description: "How Greenplum is read: each platform that loads it has its own page."
+---
+
+
diff --git a/content/docs/(moved)/connectors/hdfs.mdx b/content/docs/(moved)/connectors/hdfs.mdx
new file mode 100644
index 0000000..9989406
--- /dev/null
+++ b/content/docs/(moved)/connectors/hdfs.mdx
@@ -0,0 +1,6 @@
+---
+title: HDFS
+description: "How HDFS is read: each platform that loads it has its own page."
+---
+
+
diff --git a/content/docs/(moved)/connectors/hive.mdx b/content/docs/(moved)/connectors/hive.mdx
new file mode 100644
index 0000000..8a9e631
--- /dev/null
+++ b/content/docs/(moved)/connectors/hive.mdx
@@ -0,0 +1,6 @@
+---
+title: Hive
+description: "How Hive is read: each platform that loads it has its own page."
+---
+
+
diff --git a/content/docs/(moved)/connectors/http.mdx b/content/docs/(moved)/connectors/http.mdx
new file mode 100644
index 0000000..eeb1c45
--- /dev/null
+++ b/content/docs/(moved)/connectors/http.mdx
@@ -0,0 +1,6 @@
+---
+title: HTTP
+description: "How HTTP is read: each platform that loads it has its own page."
+---
+
+
diff --git a/content/docs/(moved)/connectors/hubspot.mdx b/content/docs/(moved)/connectors/hubspot.mdx
new file mode 100644
index 0000000..13672e4
--- /dev/null
+++ b/content/docs/(moved)/connectors/hubspot.mdx
@@ -0,0 +1,6 @@
+---
+title: HubSpot
+description: "How HubSpot is read: each platform that loads it has its own page."
+---
+
+
diff --git a/content/docs/(moved)/connectors/impala.mdx b/content/docs/(moved)/connectors/impala.mdx
new file mode 100644
index 0000000..6e09337
--- /dev/null
+++ b/content/docs/(moved)/connectors/impala.mdx
@@ -0,0 +1,6 @@
+---
+title: Apache Impala
+description: "How Apache Impala is read: each platform that loads it has its own page."
+---
+
+
diff --git a/content/docs/(moved)/connectors/index.mdx b/content/docs/(moved)/connectors/index.mdx
new file mode 100644
index 0000000..ac404a7
--- /dev/null
+++ b/content/docs/(moved)/connectors/index.mdx
@@ -0,0 +1,6 @@
+---
+title: Connectors
+description: "The sources Ingestron can read: each platform has its own list."
+---
+
+
diff --git a/content/docs/(moved)/connectors/informix.mdx b/content/docs/(moved)/connectors/informix.mdx
new file mode 100644
index 0000000..b037553
--- /dev/null
+++ b/content/docs/(moved)/connectors/informix.mdx
@@ -0,0 +1,6 @@
+---
+title: Informix
+description: "How Informix is read: each platform that loads it has its own page."
+---
+
+
diff --git a/content/docs/(moved)/connectors/jira.mdx b/content/docs/(moved)/connectors/jira.mdx
new file mode 100644
index 0000000..a12af58
--- /dev/null
+++ b/content/docs/(moved)/connectors/jira.mdx
@@ -0,0 +1,6 @@
+---
+title: Jira
+description: "How Jira is read: each platform that loads it has its own page."
+---
+
+
diff --git a/content/docs/(moved)/connectors/kafka.mdx b/content/docs/(moved)/connectors/kafka.mdx
new file mode 100644
index 0000000..6052ae2
--- /dev/null
+++ b/content/docs/(moved)/connectors/kafka.mdx
@@ -0,0 +1,6 @@
+---
+title: Kafka
+description: "How Kafka is read: each platform that loads it has its own page."
+---
+
+
diff --git a/content/docs/(moved)/connectors/mariadb.mdx b/content/docs/(moved)/connectors/mariadb.mdx
new file mode 100644
index 0000000..517cdf9
--- /dev/null
+++ b/content/docs/(moved)/connectors/mariadb.mdx
@@ -0,0 +1,6 @@
+---
+title: MariaDB
+description: "How MariaDB is read: each platform that loads it has its own page."
+---
+
+
diff --git a/content/docs/(moved)/connectors/microsoft-365.mdx b/content/docs/(moved)/connectors/microsoft-365.mdx
new file mode 100644
index 0000000..7653561
--- /dev/null
+++ b/content/docs/(moved)/connectors/microsoft-365.mdx
@@ -0,0 +1,6 @@
+---
+title: Microsoft 365
+description: "How Microsoft 365 is read: each platform that loads it has its own page."
+---
+
+
diff --git a/content/docs/(moved)/connectors/microsoft-access.mdx b/content/docs/(moved)/connectors/microsoft-access.mdx
new file mode 100644
index 0000000..b0da911
--- /dev/null
+++ b/content/docs/(moved)/connectors/microsoft-access.mdx
@@ -0,0 +1,6 @@
+---
+title: Microsoft Access
+description: "How Microsoft Access is read: each platform that loads it has its own page."
+---
+
+
diff --git a/content/docs/(moved)/connectors/mongodb-atlas.mdx b/content/docs/(moved)/connectors/mongodb-atlas.mdx
new file mode 100644
index 0000000..c824969
--- /dev/null
+++ b/content/docs/(moved)/connectors/mongodb-atlas.mdx
@@ -0,0 +1,6 @@
+---
+title: MongoDB Atlas
+description: "How MongoDB Atlas is read: each platform that loads it has its own page."
+---
+
+
diff --git a/content/docs/(moved)/connectors/mongodb.mdx b/content/docs/(moved)/connectors/mongodb.mdx
new file mode 100644
index 0000000..eb829cb
--- /dev/null
+++ b/content/docs/(moved)/connectors/mongodb.mdx
@@ -0,0 +1,6 @@
+---
+title: MongoDB
+description: "How MongoDB is read: each platform that loads it has its own page."
+---
+
+
diff --git a/content/docs/(moved)/connectors/mysql.mdx b/content/docs/(moved)/connectors/mysql.mdx
new file mode 100644
index 0000000..b172a33
--- /dev/null
+++ b/content/docs/(moved)/connectors/mysql.mdx
@@ -0,0 +1,6 @@
+---
+title: MySQL
+description: "How MySQL is read: each platform that loads it has its own page."
+---
+
+
diff --git a/content/docs/(moved)/connectors/netezza.mdx b/content/docs/(moved)/connectors/netezza.mdx
new file mode 100644
index 0000000..849b743
--- /dev/null
+++ b/content/docs/(moved)/connectors/netezza.mdx
@@ -0,0 +1,6 @@
+---
+title: Netezza
+description: "How Netezza is read: each platform that loads it has its own page."
+---
+
+
diff --git a/content/docs/(moved)/connectors/odata.mdx b/content/docs/(moved)/connectors/odata.mdx
new file mode 100644
index 0000000..f25f83a
--- /dev/null
+++ b/content/docs/(moved)/connectors/odata.mdx
@@ -0,0 +1,6 @@
+---
+title: OData
+description: "How OData is read: each platform that loads it has its own page."
+---
+
+
diff --git a/content/docs/(moved)/connectors/odbc.mdx b/content/docs/(moved)/connectors/odbc.mdx
new file mode 100644
index 0000000..360fe16
--- /dev/null
+++ b/content/docs/(moved)/connectors/odbc.mdx
@@ -0,0 +1,6 @@
+---
+title: ODBC
+description: "How ODBC is read: each platform that loads it has its own page."
+---
+
+
diff --git a/content/docs/(moved)/connectors/oracle-cloud-storage.mdx b/content/docs/(moved)/connectors/oracle-cloud-storage.mdx
new file mode 100644
index 0000000..c15d94f
--- /dev/null
+++ b/content/docs/(moved)/connectors/oracle-cloud-storage.mdx
@@ -0,0 +1,6 @@
+---
+title: Oracle Cloud Storage
+description: "How Oracle Cloud Storage is read: each platform that loads it has its own page."
+---
+
+
diff --git a/content/docs/(moved)/connectors/oracle.mdx b/content/docs/(moved)/connectors/oracle.mdx
new file mode 100644
index 0000000..35dc7bc
--- /dev/null
+++ b/content/docs/(moved)/connectors/oracle.mdx
@@ -0,0 +1,6 @@
+---
+title: Oracle
+description: "How Oracle is read: each platform that loads it has its own page."
+---
+
+
diff --git a/content/docs/(moved)/connectors/postgresql.mdx b/content/docs/(moved)/connectors/postgresql.mdx
new file mode 100644
index 0000000..9a50b09
--- /dev/null
+++ b/content/docs/(moved)/connectors/postgresql.mdx
@@ -0,0 +1,6 @@
+---
+title: PostgreSQL
+description: "How PostgreSQL is read: each platform that loads it has its own page."
+---
+
+
diff --git a/content/docs/(moved)/connectors/presto.mdx b/content/docs/(moved)/connectors/presto.mdx
new file mode 100644
index 0000000..268d0be
--- /dev/null
+++ b/content/docs/(moved)/connectors/presto.mdx
@@ -0,0 +1,6 @@
+---
+title: Presto
+description: "How Presto is read: each platform that loads it has its own page."
+---
+
+
diff --git a/content/docs/(moved)/connectors/quickbooks.mdx b/content/docs/(moved)/connectors/quickbooks.mdx
new file mode 100644
index 0000000..7b9a00a
--- /dev/null
+++ b/content/docs/(moved)/connectors/quickbooks.mdx
@@ -0,0 +1,6 @@
+---
+title: QuickBooks Online
+description: "How QuickBooks Online is read: each platform that loads it has its own page."
+---
+
+
diff --git a/content/docs/(moved)/connectors/rest.mdx b/content/docs/(moved)/connectors/rest.mdx
new file mode 100644
index 0000000..bdc0915
--- /dev/null
+++ b/content/docs/(moved)/connectors/rest.mdx
@@ -0,0 +1,6 @@
+---
+title: REST
+description: "How REST is read: each platform that loads it has its own page."
+---
+
+
diff --git a/content/docs/(moved)/connectors/s3-compatible.mdx b/content/docs/(moved)/connectors/s3-compatible.mdx
new file mode 100644
index 0000000..ca96b77
--- /dev/null
+++ b/content/docs/(moved)/connectors/s3-compatible.mdx
@@ -0,0 +1,6 @@
+---
+title: Amazon S3 Compatible Storage
+description: "How Amazon S3 Compatible Storage is read: each platform that loads it has its own page."
+---
+
+
diff --git a/content/docs/(moved)/connectors/salesforce-service-cloud.mdx b/content/docs/(moved)/connectors/salesforce-service-cloud.mdx
new file mode 100644
index 0000000..64e4920
--- /dev/null
+++ b/content/docs/(moved)/connectors/salesforce-service-cloud.mdx
@@ -0,0 +1,6 @@
+---
+title: Salesforce Service Cloud
+description: "How Salesforce Service Cloud is read: each platform that loads it has its own page."
+---
+
+
diff --git a/content/docs/(moved)/connectors/salesforce.mdx b/content/docs/(moved)/connectors/salesforce.mdx
new file mode 100644
index 0000000..54369a6
--- /dev/null
+++ b/content/docs/(moved)/connectors/salesforce.mdx
@@ -0,0 +1,6 @@
+---
+title: Salesforce
+description: "How Salesforce is read: each platform that loads it has its own page."
+---
+
+
diff --git a/content/docs/(moved)/connectors/sap-bw-mdx.mdx b/content/docs/(moved)/connectors/sap-bw-mdx.mdx
new file mode 100644
index 0000000..362cca1
--- /dev/null
+++ b/content/docs/(moved)/connectors/sap-bw-mdx.mdx
@@ -0,0 +1,6 @@
+---
+title: SAP Business Warehouse via MDX
+description: "How SAP Business Warehouse via MDX is read: each platform that loads it has its own page."
+---
+
+
diff --git a/content/docs/(moved)/connectors/sap-bw-open-hub.mdx b/content/docs/(moved)/connectors/sap-bw-open-hub.mdx
new file mode 100644
index 0000000..fe4152e
--- /dev/null
+++ b/content/docs/(moved)/connectors/sap-bw-open-hub.mdx
@@ -0,0 +1,6 @@
+---
+title: SAP Business Warehouse Open Hub
+description: "How SAP Business Warehouse Open Hub is read: each platform that loads it has its own page."
+---
+
+
diff --git a/content/docs/(moved)/connectors/sap-c4c.mdx b/content/docs/(moved)/connectors/sap-c4c.mdx
new file mode 100644
index 0000000..c24ec40
--- /dev/null
+++ b/content/docs/(moved)/connectors/sap-c4c.mdx
@@ -0,0 +1,6 @@
+---
+title: SAP Cloud for Customer
+description: "How SAP Cloud for Customer is read: each platform that loads it has its own page."
+---
+
+
diff --git a/content/docs/(moved)/connectors/sap-ecc.mdx b/content/docs/(moved)/connectors/sap-ecc.mdx
new file mode 100644
index 0000000..dfb4a9a
--- /dev/null
+++ b/content/docs/(moved)/connectors/sap-ecc.mdx
@@ -0,0 +1,6 @@
+---
+title: SAP ECC
+description: "How SAP ECC is read: each platform that loads it has its own page."
+---
+
+
diff --git a/content/docs/(moved)/connectors/sap-hana.mdx b/content/docs/(moved)/connectors/sap-hana.mdx
new file mode 100644
index 0000000..9e76ab7
--- /dev/null
+++ b/content/docs/(moved)/connectors/sap-hana.mdx
@@ -0,0 +1,6 @@
+---
+title: SAP HANA
+description: "How SAP HANA is read: each platform that loads it has its own page."
+---
+
+
diff --git a/content/docs/(moved)/connectors/sap-table.mdx b/content/docs/(moved)/connectors/sap-table.mdx
new file mode 100644
index 0000000..8ab4c79
--- /dev/null
+++ b/content/docs/(moved)/connectors/sap-table.mdx
@@ -0,0 +1,6 @@
+---
+title: SAP Table
+description: "How SAP Table is read: each platform that loads it has its own page."
+---
+
+
diff --git a/content/docs/(moved)/connectors/servicenow.mdx b/content/docs/(moved)/connectors/servicenow.mdx
new file mode 100644
index 0000000..df0fe28
--- /dev/null
+++ b/content/docs/(moved)/connectors/servicenow.mdx
@@ -0,0 +1,6 @@
+---
+title: ServiceNow
+description: "How ServiceNow is read: each platform that loads it has its own page."
+---
+
+
diff --git a/content/docs/(moved)/connectors/sftp.mdx b/content/docs/(moved)/connectors/sftp.mdx
new file mode 100644
index 0000000..20f739c
--- /dev/null
+++ b/content/docs/(moved)/connectors/sftp.mdx
@@ -0,0 +1,6 @@
+---
+title: SFTP
+description: "How SFTP is read: each platform that loads it has its own page."
+---
+
+
diff --git a/content/docs/(moved)/connectors/sharepoint-online-list.mdx b/content/docs/(moved)/connectors/sharepoint-online-list.mdx
new file mode 100644
index 0000000..b9486f7
--- /dev/null
+++ b/content/docs/(moved)/connectors/sharepoint-online-list.mdx
@@ -0,0 +1,6 @@
+---
+title: SharePoint Online List
+description: "How SharePoint Online List is read: each platform that loads it has its own page."
+---
+
+
diff --git a/content/docs/(moved)/connectors/shopify.mdx b/content/docs/(moved)/connectors/shopify.mdx
new file mode 100644
index 0000000..5279362
--- /dev/null
+++ b/content/docs/(moved)/connectors/shopify.mdx
@@ -0,0 +1,6 @@
+---
+title: Shopify
+description: "How Shopify is read: each platform that loads it has its own page."
+---
+
+
diff --git a/content/docs/(moved)/connectors/snowflake.mdx b/content/docs/(moved)/connectors/snowflake.mdx
new file mode 100644
index 0000000..537d08e
--- /dev/null
+++ b/content/docs/(moved)/connectors/snowflake.mdx
@@ -0,0 +1,6 @@
+---
+title: Snowflake
+description: "How Snowflake is read: each platform that loads it has its own page."
+---
+
+
diff --git a/content/docs/(moved)/connectors/spark.mdx b/content/docs/(moved)/connectors/spark.mdx
new file mode 100644
index 0000000..cfeee82
--- /dev/null
+++ b/content/docs/(moved)/connectors/spark.mdx
@@ -0,0 +1,6 @@
+---
+title: Spark
+description: "How Spark is read: each platform that loads it has its own page."
+---
+
+
diff --git a/content/docs/(moved)/connectors/sqlserver.mdx b/content/docs/(moved)/connectors/sqlserver.mdx
new file mode 100644
index 0000000..ceb99c2
--- /dev/null
+++ b/content/docs/(moved)/connectors/sqlserver.mdx
@@ -0,0 +1,6 @@
+---
+title: SQL Server
+description: "How SQL Server is read: each platform that loads it has its own page."
+---
+
+
diff --git a/content/docs/(moved)/connectors/square.mdx b/content/docs/(moved)/connectors/square.mdx
new file mode 100644
index 0000000..203ef56
--- /dev/null
+++ b/content/docs/(moved)/connectors/square.mdx
@@ -0,0 +1,6 @@
+---
+title: Square
+description: "How Square is read: each platform that loads it has its own page."
+---
+
+
diff --git a/content/docs/(moved)/connectors/sybase.mdx b/content/docs/(moved)/connectors/sybase.mdx
new file mode 100644
index 0000000..26d708a
--- /dev/null
+++ b/content/docs/(moved)/connectors/sybase.mdx
@@ -0,0 +1,6 @@
+---
+title: Sybase
+description: "How Sybase is read: each platform that loads it has its own page."
+---
+
+
diff --git a/content/docs/(moved)/connectors/teradata.mdx b/content/docs/(moved)/connectors/teradata.mdx
new file mode 100644
index 0000000..e63b171
--- /dev/null
+++ b/content/docs/(moved)/connectors/teradata.mdx
@@ -0,0 +1,6 @@
+---
+title: Teradata
+description: "How Teradata is read: each platform that loads it has its own page."
+---
+
+
diff --git a/content/docs/(moved)/connectors/vertica.mdx b/content/docs/(moved)/connectors/vertica.mdx
new file mode 100644
index 0000000..70205ed
--- /dev/null
+++ b/content/docs/(moved)/connectors/vertica.mdx
@@ -0,0 +1,6 @@
+---
+title: Vertica
+description: "How Vertica is read: each platform that loads it has its own page."
+---
+
+
diff --git a/content/docs/(moved)/connectors/web-table.mdx b/content/docs/(moved)/connectors/web-table.mdx
new file mode 100644
index 0000000..f4ad12f
--- /dev/null
+++ b/content/docs/(moved)/connectors/web-table.mdx
@@ -0,0 +1,6 @@
+---
+title: Web table
+description: "How Web table is read: each platform that loads it has its own page."
+---
+
+
diff --git a/content/docs/(moved)/connectors/xero.mdx b/content/docs/(moved)/connectors/xero.mdx
new file mode 100644
index 0000000..1ef4b81
--- /dev/null
+++ b/content/docs/(moved)/connectors/xero.mdx
@@ -0,0 +1,6 @@
+---
+title: Xero
+description: "How Xero is read: each platform that loads it has its own page."
+---
+
+
diff --git a/content/docs/(moved)/how-to/accept-a-warning.mdx b/content/docs/(moved)/how-to/accept-a-warning.mdx
new file mode 100644
index 0000000..cd229c6
--- /dev/null
+++ b/content/docs/(moved)/how-to/accept-a-warning.mdx
@@ -0,0 +1,6 @@
+---
+title: Accept a warning
+description: Take a warning you have looked at and decided to live with out of the problem lists, with the reason, for every environment or only some.
+---
+
+
diff --git a/content/docs/(moved)/how-to/add-a-source.mdx b/content/docs/(moved)/how-to/add-a-source.mdx
new file mode 100644
index 0000000..d63ba95
--- /dev/null
+++ b/content/docs/(moved)/how-to/add-a-source.mdx
@@ -0,0 +1,6 @@
+---
+title: Add a source and its tables
+description: Grow a project after you create it with another table, another source, or a first source beside what you already run.
+---
+
+
diff --git a/content/docs/(moved)/how-to/add-an-environment.mdx b/content/docs/(moved)/how-to/add-an-environment.mdx
new file mode 100644
index 0000000..942c522
--- /dev/null
+++ b/content/docs/(moved)/how-to/add-an-environment.mdx
@@ -0,0 +1,6 @@
+---
+title: Add an environment
+description: Add another environment, such as test, to the files every environment shares, and see what it changes.
+---
+
+
diff --git a/content/docs/(moved)/how-to/add-quality-checks.mdx b/content/docs/(moved)/how-to/add-quality-checks.mdx
new file mode 100644
index 0000000..7f26da2
--- /dev/null
+++ b/content/docs/(moved)/how-to/add-quality-checks.mdx
@@ -0,0 +1,6 @@
+---
+title: Check data quality
+description: Count, leave out or stop on rows that break a rule, in the layers that check, on Databricks, DuckDB and Synapse.
+---
+
+
diff --git a/content/docs/(moved)/how-to/ask-the-platform-team.mdx b/content/docs/(moved)/how-to/ask-the-platform-team.mdx
new file mode 100644
index 0000000..ddabd85
--- /dev/null
+++ b/content/docs/(moved)/how-to/ask-the-platform-team.mdx
@@ -0,0 +1,6 @@
+---
+title: Ask the platform team
+description: Write one request per environment with what the project needs from whoever runs the data platform, including the values still to provide.
+---
+
+
diff --git a/content/docs/(moved)/how-to/check-in-ci.mdx b/content/docs/(moved)/how-to/check-in-ci.mdx
new file mode 100644
index 0000000..9ce4a3e
--- /dev/null
+++ b/content/docs/(moved)/how-to/check-in-ci.mdx
@@ -0,0 +1,6 @@
+---
+title: Check generated files in CI
+description: Fail a pull request when someone changed the definitions without generating, with GitHub Actions or Azure DevOps.
+---
+
+
diff --git a/content/docs/(moved)/how-to/choose-a-published-standard.mdx b/content/docs/(moved)/how-to/choose-a-published-standard.mdx
new file mode 100644
index 0000000..bf774b2
--- /dev/null
+++ b/content/docs/(moved)/how-to/choose-a-published-standard.mdx
@@ -0,0 +1,6 @@
+---
+title: Choose a published standard
+description: Pick a dated standard for your platform, see what it decides, and list the ones there are.
+---
+
+
diff --git a/content/docs/(moved)/how-to/cleanse-data.mdx b/content/docs/(moved)/how-to/cleanse-data.mdx
new file mode 100644
index 0000000..bba94c1
--- /dev/null
+++ b/content/docs/(moved)/how-to/cleanse-data.mdx
@@ -0,0 +1,6 @@
+---
+title: Cleanse data in the right layer
+description: Keep raw rows in bronze, and trim, map and default text and bring timestamps to UTC in silver, with a transform that needs no code.
+---
+
+
diff --git a/content/docs/(moved)/how-to/configure-a-standard.mdx b/content/docs/(moved)/how-to/configure-a-standard.mdx
new file mode 100644
index 0000000..70dbcb5
--- /dev/null
+++ b/content/docs/(moved)/how-to/configure-a-standard.mdx
@@ -0,0 +1,6 @@
+---
+title: Set your organisation's standard
+description: Write one standard every project follows, start each project from it, and keep projects to it.
+---
+
+
diff --git a/content/docs/(moved)/how-to/define-a-data-product.mdx b/content/docs/(moved)/how-to/define-a-data-product.mdx
new file mode 100644
index 0000000..3267f26
--- /dev/null
+++ b/content/docs/(moved)/how-to/define-a-data-product.mdx
@@ -0,0 +1,6 @@
+---
+title: Define a data product
+description: Declare who owns a data product and what it publishes, require governance in the standard, and get a passport per product.
+---
+
+
diff --git a/content/docs/(moved)/how-to/deploy-through-environments.mdx b/content/docs/(moved)/how-to/deploy-through-environments.mdx
new file mode 100644
index 0000000..f04ad76
--- /dev/null
+++ b/content/docs/(moved)/how-to/deploy-through-environments.mdx
@@ -0,0 +1,6 @@
+---
+title: Deploy through environments
+description: Promote one commit's generated files from dev to test to prod, each stage after approval, with the GitHub Actions or Azure DevOps templates.
+---
+
+
diff --git a/content/docs/(moved)/how-to/describe-a-source.mdx b/content/docs/(moved)/how-to/describe-a-source.mdx
new file mode 100644
index 0000000..4b77a64
--- /dev/null
+++ b/content/docs/(moved)/how-to/describe-a-source.mdx
@@ -0,0 +1,6 @@
+---
+title: Describe a source
+description: Get each source table's columns, types and key into the project without connecting to the source, see what changed at the source, and draft the contracts that are needed.
+---
+
+
diff --git a/content/docs/(moved)/how-to/grant-access.mdx b/content/docs/(moved)/how-to/grant-access.mdx
new file mode 100644
index 0000000..6f72728
--- /dev/null
+++ b/content/docs/(moved)/how-to/grant-access.mdx
@@ -0,0 +1,6 @@
+---
+title: Grant access, mask columns and filter rows
+description: Say who may read each table, hide sensitive columns and filter rows, and have Ingestron grant, mask and filter on Databricks, grant on Synapse, and say what an engine cannot do.
+---
+
+
diff --git a/content/docs/(moved)/how-to/keep-a-table-current.mdx b/content/docs/(moved)/how-to/keep-a-table-current.mdx
new file mode 100644
index 0000000..b534e7d
--- /dev/null
+++ b/content/docs/(moved)/how-to/keep-a-table-current.mdx
@@ -0,0 +1,6 @@
+---
+title: Keep a table current
+description: Keep the current row for each key, or every version, from loads of changed rows on DuckDB or Databricks.
+---
+
+
diff --git a/content/docs/(moved)/how-to/name-things-your-way.mdx b/content/docs/(moved)/how-to/name-things-your-way.mdx
new file mode 100644
index 0000000..028d405
--- /dev/null
+++ b/content/docs/(moved)/how-to/name-things-your-way.mdx
@@ -0,0 +1,6 @@
+---
+title: Name tables and resources
+description: Set the catalog, schema, table, pipeline and job names every project gets, name ADF's resources, and see the names before anything is deployed.
+---
+
+
diff --git a/content/docs/(moved)/how-to/plan-and-generate.mdx b/content/docs/(moved)/how-to/plan-and-generate.mdx
new file mode 100644
index 0000000..def6a0a
--- /dev/null
+++ b/content/docs/(moved)/how-to/plan-and-generate.mdx
@@ -0,0 +1,6 @@
+---
+title: Plan, apply and review
+description: Validate definitions, inspect proposed file changes and write the reviewed output.
+---
+
+
diff --git a/content/docs/(moved)/how-to/run-a-flow-on-its-own-triggers.mdx b/content/docs/(moved)/how-to/run-a-flow-on-its-own-triggers.mdx
new file mode 100644
index 0000000..bf4c7d9
--- /dev/null
+++ b/content/docs/(moved)/how-to/run-a-flow-on-its-own-triggers.mdx
@@ -0,0 +1,6 @@
+---
+title: Run a flow on its own triggers
+description: Write an hourly window and a weekly full reload for one flow, inside the hours your standard allows, on Azure Data Factory or Synapse.
+---
+
+
diff --git a/content/docs/(moved)/how-to/share-a-template.mdx b/content/docs/(moved)/how-to/share-a-template.mdx
new file mode 100644
index 0000000..96813df
--- /dev/null
+++ b/content/docs/(moved)/how-to/share-a-template.mdx
@@ -0,0 +1,6 @@
+---
+title: Share templates and standards
+description: Keep your team's project templates, your organisation's standard and your own published standards in Git, and start projects from them.
+---
+
+
diff --git a/content/docs/(moved)/how-to/start-from-a-published-standard.mdx b/content/docs/(moved)/how-to/start-from-a-published-standard.mdx
new file mode 100644
index 0000000..e5a9fe1
--- /dev/null
+++ b/content/docs/(moved)/how-to/start-from-a-published-standard.mdx
@@ -0,0 +1,6 @@
+---
+title: Start from a published standard
+description: Create a project from a published standard, write the values it leaves to you, and plan.
+---
+
+
diff --git a/content/docs/(moved)/how-to/start-from-a-template.mdx b/content/docs/(moved)/how-to/start-from-a-template.mdx
new file mode 100644
index 0000000..5255ed4
--- /dev/null
+++ b/content/docs/(moved)/how-to/start-from-a-template.mdx
@@ -0,0 +1,6 @@
+---
+title: Create a project
+description: Define a standard and target platform, then add connections and flows independently.
+---
+
+
diff --git a/content/docs/(moved)/how-to/upgrade-a-standard.mdx b/content/docs/(moved)/how-to/upgrade-a-standard.mdx
new file mode 100644
index 0000000..5e7ff01
--- /dev/null
+++ b/content/docs/(moved)/how-to/upgrade-a-standard.mdx
@@ -0,0 +1,6 @@
+---
+title: Move to a newer published standard
+description: Replace a project's standard with a later version of its published standard, see what changes with plan and git diff, and keep your own edits.
+---
+
+
diff --git a/content/docs/(moved)/how-to/upgrade.mdx b/content/docs/(moved)/how-to/upgrade.mdx
new file mode 100644
index 0000000..cfe8b66
--- /dev/null
+++ b/content/docs/(moved)/how-to/upgrade.mdx
@@ -0,0 +1,6 @@
+---
+title: Upgrade Ingestron and its packs
+description: Choose the version every project runs, see what an upgrade changes with plan, move CI and your team together, and move from 1.x to 2.0.
+---
+
+
diff --git a/content/docs/(moved)/how-to/write-a-contract.mdx b/content/docs/(moved)/how-to/write-a-contract.mdx
new file mode 100644
index 0000000..a5aa971
--- /dev/null
+++ b/content/docs/(moved)/how-to/write-a-contract.mdx
@@ -0,0 +1,6 @@
+---
+title: Describe a table with a contract
+description: Write a table's columns, types, keys and rules in a data contract, or draft one from a sample file or from the source's metadata.
+---
+
+
diff --git a/content/docs/(moved)/how-to/write-a-transform.mdx b/content/docs/(moved)/how-to/write-a-transform.mdx
new file mode 100644
index 0000000..f71da72
--- /dev/null
+++ b/content/docs/(moved)/how-to/write-a-transform.mdx
@@ -0,0 +1,6 @@
+---
+title: Write a transform
+description: Build a table from other tables with SQL on DuckDB, Synapse or Databricks, or with PySpark on Databricks.
+---
+
+
diff --git a/content/docs/(moved)/meta.json b/content/docs/(moved)/meta.json
new file mode 100644
index 0000000..fce7505
--- /dev/null
+++ b/content/docs/(moved)/meta.json
@@ -0,0 +1,6 @@
+{
+ "title": "Choose your platform",
+ "description": "A page's old URL: it links to the page's version for each platform.",
+ "root": true,
+ "pages": ["..."]
+}
diff --git a/content/docs/(moved)/start/first-project.mdx b/content/docs/(moved)/start/first-project.mdx
new file mode 100644
index 0000000..f3744bb
--- /dev/null
+++ b/content/docs/(moved)/start/first-project.mdx
@@ -0,0 +1,6 @@
+---
+title: Run your first project on your laptop
+description: Create a project from the DuckDB starter, generate its SQL, and build two tables on your laptop. No cloud account, no source, nothing deployed.
+---
+
+
diff --git a/content/docs/(moved)/start/how-it-works.mdx b/content/docs/(moved)/start/how-it-works.mdx
new file mode 100644
index 0000000..fee381e
--- /dev/null
+++ b/content/docs/(moved)/start/how-it-works.mdx
@@ -0,0 +1,6 @@
+---
+title: How Ingestron works
+description: What you write, what Ingestron writes from it, and what runs on your platform afterwards.
+---
+
+
diff --git a/content/docs/(moved)/start/install.mdx b/content/docs/(moved)/start/install.mdx
new file mode 100644
index 0000000..31874a3
--- /dev/null
+++ b/content/docs/(moved)/start/install.mdx
@@ -0,0 +1,6 @@
+---
+title: Install the command and the extension
+description: Install the ingestron command, the VS Code extension, or both, to write, plan and generate projects on your computer.
+---
+
+
diff --git a/content/docs/(reference)/reference/cli.mdx b/content/docs/(reference)/reference/cli.mdx
index 48ec962..9f39baf 100644
--- a/content/docs/(reference)/reference/cli.mdx
+++ b/content/docs/(reference)/reference/cli.mdx
@@ -93,7 +93,9 @@ table named from the imported metadata, and with `--needed`, for every table
that needs a contract and names none. It resolves for the environment `--env`
names, or the first one. It writes `contracts/.
.yaml`,
never replaces a file that is there, and prints the `contract:` line to add to
-the flow. A draft says what to review: its types, which columns are required
+the flow. When no import matches a new contract, it adds `contracts/*.yaml` to
+`imports`, or the contract's own path when `contracts/` holds other YAML files,
+and says so. A draft says what to review: its types, which columns are required
and its key.
The document an import writes is in [Documents](/docs/reference/documents#source-metadata),
diff --git a/content/docs/(reference)/reference/documents.mdx b/content/docs/(reference)/reference/documents.mdx
index 9e59c40..c76c90d 100644
--- a/content/docs/(reference)/reference/documents.mdx
+++ b/content/docs/(reference)/reference/documents.mdx
@@ -59,7 +59,7 @@ for `principals`.
-See [Connectors](/docs/connectors) and [Reachability](/docs/start/choose-a-route#reachability).
+See [Connectors](/docs/connectors) and [Reachability](/docs/concepts/choose-a-platform#reachability).
A connection's `timezone` is the zone its timestamps are converted from; see
[Cleanse data in the right layer](/docs/how-to/cleanse-data#add-the-columns-to-cleanse).
diff --git a/content/docs/(reference)/reference/errors.mdx b/content/docs/(reference)/reference/errors.mdx
index 2957a17..e2e5405 100644
--- a/content/docs/(reference)/reference/errors.mdx
+++ b/content/docs/(reference)/reference/errors.mdx
@@ -31,3 +31,14 @@ These come from reading the YAML files and resolving `${…}` references,
before the definitions are checked.
+
+## Problems with the output folder [#output-folder]
+
+These are about the files Ingestron would write, not about the definitions.
+`plan` finds `BAD_PATH`, `BAD_MANIFEST` and `FIRST_WRITE` and shows each under
+the files of the output folder it concerns, as `problem `. A plan with
+a problem is not clean: `plan` exits with `1` and `apply` writes nothing to
+that folder. The other codes are `apply`'s, found as it writes. Each entry says
+which command reports it.
+
+
diff --git a/content/docs/(reference)/reference/intent.mdx b/content/docs/(reference)/reference/intent.mdx
index 31fcb4b..0f88659 100644
--- a/content/docs/(reference)/reference/intent.mdx
+++ b/content/docs/(reference)/reference/intent.mdx
@@ -9,7 +9,7 @@ pack turns it into its platform's terms, and says what it cannot do. Every
part is optional: a project that writes none builds as it did before.
This is the standard of the
-[local medallion](/docs/how-to/kits/duckdb-local-medallion) published
+[local medallion](/docs/duckdb/local-medallion) published
standard. Its `layers`, with what each layer `does`, `load`, `cleanse`,
`quality` and `features` are intent; `ingest` is a method setting:
@@ -60,7 +60,7 @@ name, is [`BAD_SETTING`](/docs/reference/errors#BAD_SETTING).
another value is [`FIXED_SETTING`](/docs/reference/errors#FIXED_SETTING).
This flow of the
-[streaming lakehouse](/docs/how-to/kits/databricks-lakehouse-streaming)
+[streaming lakehouse](/docs/databricks/lakehouse-streaming)
published standard says which change to an order is newest, and which change
is a delete:
@@ -201,13 +201,19 @@ Without `features`, nothing is reported. Published standards say
`operations` is the policy for running what is generated: one schedule for
the project (`schedule: { per: project }`), what starts a run (`trigger`),
-`retries`, a `timeout`, `alerts` and `concurrency`. Without it, the project
+`retries`, a `timeout`, `alerts` and `concurrency`. With `schedule: { per: flow }`
+and `triggers`, flows write their own Data Factory or Synapse triggers, within
+the trigger types, time zone and hours the standard sets; see
+[Run a flow on its own triggers](/docs/how-to/run-a-flow-on-its-own-triggers). Without it, the project
runs on one schedule, and the platform's alerts are sent when a run fails. Values that differ per
environment belong in the platform: the cron and time zone in `schedule`, who
is told in `alerts`, and the tags in `tags`. `require_tags` names tags the
platform must give; one missing is
[`BAD_SETTING`](/docs/reference/errors#BAD_SETTING) at the platform's `tags`.
-The first pack in the standard's list that runs jobs applies `operations`:
-the Databricks refresh job, or the Data Factory triggers and copies. When no
-pack of the standard runs jobs, each value is `NOT_APPLIED`.
+Each `operations` value is applied by the first pack in the standard's list
+that applies it: the Databricks refresh job, or the Data Factory triggers and
+copies. With `packs: [databricks@2, adf@2]`, Data Factory applies
+`schedule: { per: flow }`, which the Databricks job does not. A value no pack
+applies is judged by the first pack that runs jobs. When no pack of the
+standard runs jobs, each value is `NOT_APPLIED`.
diff --git a/content/docs/(reference)/reference/meta.json b/content/docs/(reference)/reference/meta.json
index 6b5f919..066fe9f 100644
--- a/content/docs/(reference)/reference/meta.json
+++ b/content/docs/(reference)/reference/meta.json
@@ -1,4 +1,14 @@
{
"title": "Reference",
- "pages": ["documents", "intent", "packs", "errors", "cli", "recipes", "templates", "versions"]
+ "pages": [
+ "documents",
+ "intent",
+ "packs",
+ "errors",
+ "cli",
+ "recipes",
+ "templates",
+ "versions",
+ "release-notes"
+ ]
}
diff --git a/content/docs/(reference)/reference/packs/adf.mdx b/content/docs/(reference)/reference/packs/adf.mdx
index fe10cff..debf901 100644
--- a/content/docs/(reference)/reference/packs/adf.mdx
+++ b/content/docs/(reference)/reference/packs/adf.mdx
@@ -12,19 +12,19 @@ copies as they are, and Synapse cannot read the JSON Microsoft 365 lands. List
the other pack first, as in `packs: [databricks@2, adf@2]`; see
[Combine packs](/docs/concepts/standards-and-packs#combine-packs-when-their-actions-fit).
On its own, `packs: [adf@2]` with `store: { default: none }` ends each table at
-the landed files; see [Land files only](/docs/platforms/landing-only). The
+the landed files; see [Land files only](/docs/adf/land-files-only). The
settings below come from the pack itself.
-For a task walkthrough, use [Azure Data Factory](/docs/platforms/adf),
+For a task walkthrough, use [Azure Data Factory](/docs/adf),
[Set your organisation's standard](/docs/how-to/configure-a-standard),
-[Shape ADF pipelines](/docs/platforms/adf-pipelines) or
+[Shape ADF pipelines](/docs/adf/pipelines) or
[ADF resource names](/docs/how-to/name-things-your-way#adf-resource-names).
The reference below keeps the method and setting anchors used by editor help.
The pack writes one factory for every environment, with a parameter file per
-environment; see [What Ingestron writes](/docs/platforms/adf#what-you-get).
+environment; see [What Ingestron writes](/docs/adf#what-you-get).
After generation and review,
-[deploy it as an ARM template](/docs/platforms/adf-deploy#deploy). Generation itself
+[deploy it as an ARM template](/docs/adf/deploy#deploy). Generation itself
deploys nothing.
@@ -86,6 +86,64 @@ no access to judge, unless a view serves it.
+## Triggers a flow writes [#triggers]
+
+A flow's `ingest.triggers` and `ingest.parameters` are fields of the
+[Flow document](/docs/reference/documents#flow); the standard's rules are
+under `operations.triggers` in the
+[Standard document](/docs/reference/documents#standard). For the task, see
+[Run a flow on its own triggers](/docs/how-to/run-a-flow-on-its-own-triggers).
+What the pack writes:
+
+- A **schedule** trigger is a `ScheduleTrigger` with one `pipelines` entry,
+ whose `parameters` is always present. Its `recurrence` has `frequency`
+ (`Minute`, `Hour`, `Day`, `Week` or `Month`), `interval` (1 to 1000),
+ `startTime`, `endTime` when written, `timeZone` and, from `at`, a `schedule`
+ of `weekDays`, `monthDays`, `monthlyOccurrences`, `hours` and `minutes`.
+ `timeZone` is the Windows name of the trigger's IANA zone, such as
+ `GMT Standard Time` for `Europe/London`; a zone with no Windows name is
+ refused. In UTC the times end in `Z`; in any other zone they do not.
+- A **tumbling-window** trigger is a `TumblingWindowTrigger` on exactly one
+ pipeline, in UTC with no `timeZone`, by `Minute`, `Hour` or `Month`, with
+ windows of at least 5 minutes. `delay` is `hh:mm:ss`. `maxConcurrency` (1 to 50) is the trigger's `concurrency`, else the standard's
+ `operations.concurrency.max_runs`, else 1. `retryPolicy` is its `retries`,
+ else the standard's `operations.retries`, with an interval of at least 30
+ seconds. It passes `windowStart` and `windowEnd` from
+ `@trigger().outputs.windowStartTime` and `windowEndTime`. A dependency on
+ itself needs a negative `offset`, and a trigger depends on at most five
+ others. The flow's triggers carry the annotation `ingestron:flow-trigger`,
+ and `ingestron:backfill` when they write `backfill: true`.
+- A schedule trigger passes `windowStart` and `windowEnd` too: from its
+ `lookback`, the window `[scheduledTime − lookback, scheduledTime)`, and with
+ `load: full` or no lookback both are `@trigger().scheduledTime`.
+- Each trigger's `parameters` are passed as written: a literal as it is, and
+ `{ expression }` as an ADF expression of type `Expression`. The system
+ variables an expression may use are `@trigger().scheduledTime` and
+ `@trigger().startTime` on both kinds, and
+ `@trigger().outputs.windowStartTime` and `windowEndTime` on a tumbling window.
+- A trigger is named `tr__`, and a tumbling window
+ `tr___`, such as `tr_erp_erp_erp_hourly_h1`, because ADF
+ cannot edit a published window's frequency or interval.
+- Pipeline parameters are typed `String` (`string`, `date`, `datetime`), `Int`,
+ `Float` or `Bool`. The source `filter` is written as an expression with each
+ `{{ parameters. }}` rendered as a typed, escaped value.
+
+Where Microsoft's pages disagree, the pack takes the stricter reading: a
+flow's tumbling window is `Minute`, `Hour` or `Month` only, `hours` are 0 to 23
+and `minutes` 0 to 59, and `interval` is 1 to 1000 for both kinds. The default
+trigger keeps every value it accepted before. A local time the clocks skip
+runs after the change, and one they repeat runs once, at its first
+occurrence; the pages do not say, so the allowed-hours check assumes it.
+
+The properties come from Microsoft's pages on
+[schedule triggers](https://learn.microsoft.com/en-us/azure/data-factory/how-to-create-schedule-trigger),
+[tumbling window triggers](https://learn.microsoft.com/en-us/azure/data-factory/how-to-create-tumbling-window-trigger),
+[pipeline execution and triggers](https://learn.microsoft.com/en-us/azure/data-factory/concepts-pipeline-execution-triggers),
+[tumbling window dependencies](https://learn.microsoft.com/en-us/azure/data-factory/tumbling-window-trigger-dependency)
+and [system variables](https://learn.microsoft.com/en-us/azure/data-factory/control-flow-system-variables),
+read on **15 November 2026**. The Windows zone names come from the Unicode
+CLDR's `windowsZones.xml`, read on **17 November 2026**.
+
## Native configuration sources
@@ -93,7 +151,7 @@ no access to judge, unless a view serves it.
The native properties and current connector versions were checked against these
Microsoft references on **28 October 2026**. Connection fields and method settings
-are rendered from pack metadata; see [Azure Data Factory](/docs/platforms/adf).
+are rendered from pack metadata; see [Azure Data Factory](/docs/adf).
- [ADF naming rules](https://learn.microsoft.com/en-us/azure/data-factory/naming-rules)
- [Source control and publishing](https://learn.microsoft.com/en-us/azure/data-factory/source-control)
diff --git a/content/docs/(reference)/reference/packs/databricks.mdx b/content/docs/(reference)/reference/packs/databricks.mdx
index 62003f4..7bda2dc 100644
--- a/content/docs/(reference)/reference/packs/databricks.mdx
+++ b/content/docs/(reference)/reference/packs/databricks.mdx
@@ -7,11 +7,11 @@ description: Every method of the databricks pack, with its settings and defaults
target per environment: Lakeflow pipelines with SQL and Python files,
Lakeflow Connect ingestion, and a refresh job when the platform has a schedule
or a task runs after the refresh. See
-[What Ingestron writes](/docs/platforms/databricks#what-you-get).
+[What Ingestron writes](/docs/databricks#what-you-get).
The settings below come from the pack itself.
Before deployment, [plan and review](/docs/how-to/plan-and-generate) the files,
-then [deploy the generated bundle](/docs/platforms/databricks-deploy#deploy).
+then [deploy the generated bundle](/docs/databricks/deploy#deploy).
diff --git a/content/docs/(reference)/reference/packs/duckdb.mdx b/content/docs/(reference)/reference/packs/duckdb.mdx
index 86729c6..949bafe 100644
--- a/content/docs/(reference)/reference/packs/duckdb.mdx
+++ b/content/docs/(reference)/reference/packs/duckdb.mdx
@@ -8,7 +8,7 @@ publishes each table as Parquet, with SQL that DuckDB runs on a laptop or
wherever DuckDB runs. List it first, as in `packs: [duckdb@2, adf@2]` when
ADF lands the files; see [Combine packs](/docs/concepts/standards-and-packs#combine-packs-when-their-actions-fit). The settings below come from the pack itself.
-For a task walkthrough, use [DuckDB](/docs/platforms/duckdb); the modes are
+For a task walkthrough, use [DuckDB](/docs/duckdb); the modes are
explained in [Keep a table current](/docs/how-to/keep-a-table-current#on-duckdb).
The pack writes `run.sql`, every table in
order, and `tables/`, each table's SQL alone, in a folder for each
diff --git a/content/docs/(reference)/reference/packs/index.mdx b/content/docs/(reference)/reference/packs/index.mdx
index cbf0be9..033654d 100644
--- a/content/docs/(reference)/reference/packs/index.mdx
+++ b/content/docs/(reference)/reference/packs/index.mdx
@@ -9,12 +9,12 @@ defaults, rendered from the pack itself.
## Available packs
-| Pack | Responsibilities | Generated output, for every environment |
-| ----------------------------------------------------------------- | -------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------- |
-| [Databricks](/docs/reference/packs/databricks) (`databricks@2`) | Supported direct ingestion, transforms and storage | [One bundle with a target per environment](/docs/platforms/databricks#what-you-get), in `build/databricks/` |
-| [Azure Data Factory](/docs/reference/packs/adf) (`adf@2`) | Native copying from supported sources to landing storage | [One factory with a parameter file per environment](/docs/platforms/adf#what-you-get), in `build/adf/` |
-| [Synapse serverless](/docs/reference/packs/synapse) (`synapse@2`) | SQL views over landed files and transforms | [`views.sql` with a script per environment](/docs/platforms/synapse#what-you-get), in `build/synapse/` |
-| [DuckDB](/docs/reference/packs/duckdb) (`duckdb@2`) | Tables, transforms and local runs, published as Parquet | [`run.sql` and each table's SQL](/docs/platforms/duckdb#what-you-get), in `build/duckdb//` |
+| Pack | Responsibilities | Generated output, for every environment |
+| ----------------------------------------------------------------- | -------------------------------------------------------- | ------------------------------------------------------------------------------------------------- |
+| [Databricks](/docs/reference/packs/databricks) (`databricks@2`) | Supported direct ingestion, transforms and storage | [One bundle with a target per environment](/docs/databricks#what-you-get), in `build/databricks/` |
+| [Azure Data Factory](/docs/reference/packs/adf) (`adf@2`) | Native copying from supported sources to landing storage | [One factory with a parameter file per environment](/docs/adf#what-you-get), in `build/adf/` |
+| [Synapse serverless](/docs/reference/packs/synapse) (`synapse@2`) | SQL views over landed files and transforms | [`views.sql` with a script per environment](/docs/synapse#what-you-get), in `build/synapse/` |
+| [DuckDB](/docs/reference/packs/duckdb) (`duckdb@2`) | Tables, transforms and local runs, published as Parquet | [`run.sql` and each table's SQL](/docs/duckdb#what-you-get), in `build/duckdb//` |
Connector-specific access and ingestion requirements are in
[Connectors](/docs/connectors).
@@ -26,7 +26,7 @@ work together in the combinations the routes offer.
## Which packs a project uses
-[Choose where your data goes](/docs/start/choose-a-route) lists the packs
+[Compare the platforms](/docs/concepts/choose-a-platform) lists the packs
each route needs, and what runs after each load. How a standard selects methods,
and how Ingestron checks that they fit together, is in
[Standards, packs and methods](/docs/concepts/standards-and-packs).
diff --git a/content/docs/(reference)/reference/packs/synapse.mdx b/content/docs/(reference)/reference/packs/synapse.mdx
index bc090bf..80f221d 100644
--- a/content/docs/(reference)/reference/packs/synapse.mdx
+++ b/content/docs/(reference)/reference/packs/synapse.mdx
@@ -9,17 +9,37 @@ files a store pack keeps. For views only, list it first:
`packs: [synapse@2, adf@2]`; see [Combine packs](/docs/concepts/standards-and-packs#combine-packs-when-their-actions-fit). The settings below come from the
pack itself.
-For a task walkthrough, use [Synapse serverless views](/docs/platforms/synapse).
+For a task walkthrough, use [Synapse serverless views](/docs/synapse).
+For Azure SQL full snapshots in the workspace, use `packs: [synapse@2]` and the `synapse-copy` ingestion method. The supported route generates native workspace artifacts and SQL query review files for each environment; it references an existing Azure runtime. See [Synapse pipelines and serverless views](/docs/synapse) for its limits.
The pack writes one `views.sql` for every environment, with
`environments/.sql` setting each environment's values; run that
script with sqlcmd when the definitions change. Where a table's layer checks
its rows, it also writes `checks.sql`, with `checks/.sql` for each
environment; run that after each landing to count the checks. See
-[What Ingestron writes](/docs/platforms/synapse#what-you-get). Generation itself
+[What Ingestron writes](/docs/synapse#what-you-get). Generation itself
creates nothing in Azure.
+## Triggers a flow writes [#triggers]
+
+A flow of a `synapse-copy` table may write `ingest.triggers`, `ingest.parameters`
+and a table `ingest.filter`, as for
+[the ADF pack](/docs/reference/packs/adf#triggers); see
+[Run a flow on its own triggers](/docs/how-to/run-a-flow-on-its-own-triggers).
+The route reuses that output, under `workspace//trigger/`, with the
+same names, parameters and Windows time zone. What differs:
+
+- The default trigger, the one a flow that writes none gets, is a schedule
+ trigger in UTC with the same frequency, interval and start, and passes no
+ parameters. The standard's `operations.triggers` judges it as a schedule.
+- Every pipeline has concurrency 1, so `operations.concurrency` does not apply.
+- The route copies full snapshots only. A trigger that sets `load: incremental`,
+ or a tumbling window on an incremental table, is a method check problem.
+- A new tumbling window that starts in the past is refused with
+ [`FIRST_WRITE`](/docs/reference/errors#FIRST_WRITE), unless it writes
+ `backfill: true`.
+
## Platform configuration
### The `synapse:` section [#platform]
@@ -84,7 +104,7 @@ so its SQL is checked against those pages.
view leaves out included, and table checks over the rows the view keeps. It
returns one row per check, then raises an error for the first `fail` check
that fails. It checks nothing unless something runs it after each landing;
- see [Run and operate](/docs/platforms/synapse#operate). Table checks need
+ see [Run and operate](/docs/synapse/deploy#operate). Table checks need
nothing turned on: the pack has no `checks` capability.
- **Text comparisons** ignore spaces at the end of a value, as T-SQL does:
`'GB '` equals `'GB'`. With the standard's `trim` first this makes no
diff --git a/content/docs/(reference)/reference/release-notes.mdx b/content/docs/(reference)/reference/release-notes.mdx
new file mode 100644
index 0000000..ff2e657
--- /dev/null
+++ b/content/docs/(reference)/reference/release-notes.mdx
@@ -0,0 +1,342 @@
+---
+title: Release notes
+description: What each Ingestron release since 2.0 changes in the files it generates and in the definitions, and what to do about it.
+---
+
+Upgrade on purpose, in a pull request, as
+[Upgrade Ingestron and its packs](/docs/how-to/upgrade) describes. Coming
+from 1.x, read [Upgrade from 1.x to 2.0](/docs/how-to/upgrade#upgrade-to-2)
+first.
+
+## Ingestron 2.1 [#upgrade-to-2-1]
+
+2.1 changes `init`, what `check` finds, and Hive connections on Azure Data
+Factory:
+
+- `init` asks only what picks the project: the starter, or the connector,
+ connectivity and route. It writes `REPLACE_ME`, with a comment, for each
+ value it cannot know, writes `README.md` with where to find each one, and
+ `--set =` writes one as the project is created; see
+ [Create a project](/docs/how-to/start-from-a-template#write-the-values).
+ A script that answered `init`'s questions on standard input gives `--set`
+ instead.
+- A value still `REPLACE_ME` is a [`REPLACE_ME`](/docs/reference/errors#REPLACE_ME)
+ warning in `plan` and `apply`, and an error in `check`. Projects created
+ before 2.1 have none.
+- `ingestron request` opens with **Values to replace**, and its **Values to
+ provide** says the deploying pipeline supplies each value under the name
+ given.
+- VS Code's **New project** asks no name, environments or standard, and
+ opens the new `README.md`.
+- On Azure Data Factory, a [Hive](/docs/connectors/hive) connection's
+ `transport` now decides how it signs in. `Binary` connects anonymously: the
+ linked service is written with `authenticationType: Anonymous` and no user
+ or password, and a `Binary` connection that gives `user` or `password` is
+ refused. `SASL` and `HTTP` sign in with a user and password, and need both.
+ In a 2.0 project, on `Binary`, remove `user` and `password`, or set
+ `transport` to the one your server uses; on `SASL` or `HTTP`, give both,
+ with the password as a secret such as `${secret:hive-password}`. Then run
+ `ingestron plan` to see the linked service change.
+- For code that calls the packages: `@ingestron/definitions` 1.11 adds the
+ `REPLACE_ME` code and the `replaceMe` resolve option, which `check` sets to
+ `error`; and the CLI's `Io` gains an optional `now()`, the day `init`
+ writes.
+
+## Ingestron 2.2 [#upgrade-to-2-2]
+
+2.2 generates, prints and checks exactly what 2.1 does, and the VS Code
+extension works as before. Only code that calls the packages changes:
+
+- `@ingestron/core` 1.0 is the API the command is built on.
+ `@ingestron/cli` now exports only `run`, `commands` and `flags`. Import
+ everything else it exported in 2.1, such as `generatedOutputs`,
+ `projectRoutes`, `platformRequest` and `packs`, from `@ingestron/core`,
+ under the same names. The
+ [`@ingestron/core` README](https://github.com/ingestron/ingestron/tree/main/packages/core)
+ describes them.
+- In `@ingestron/authoring`, open a project with `ProjectSnapshot.open(folder)`
+ in place of `openProject`, and call the snapshot's methods. Each function
+ that took the project first is now a method with the same name:
+ `problems(project, "dev")` becomes `snapshot.problems("dev")`. Three are
+ renamed: `projectSymbols` is `symbols`, `compareEnvironments` is `compare`
+ and `platformRequest` is `request`. The
+ [`@ingestron/authoring` README](https://github.com/ingestron/ingestron/tree/main/packages/authoring)
+ shows an example.
+
+## Ingestron 2.3 [#upgrade-to-2-3]
+
+2.3 generates the same files as 2.2 for a project that changes nothing,
+apart from a data product's passport, whose Quality and Who may use it
+sections are rewritten, and one fix on Databricks (below). It adds
+[intent](/docs/reference/intent),
+[published standards](/docs/how-to/choose-a-published-standard), layers that
+say what they do, checks on every engine, cleansing, accepted warnings, and
+grants, masks and row filters. It asks for some settings to be written in
+their new places, and some warnings have new codes:
+
+- **Settings that are intent.** `streaming-table`'s `mode`, `sequence_by`,
+ `delete_when` and `cluster_by`, `duckdb-table`'s `mode` and `sequence_by`,
+ and `adf-copy`'s `mode` and `watermark_column` mean the same as
+ [intent](/docs/reference/intent#settings-that-are-intent), such as
+ `history: latest` for `mode: upsert`. They still work, and `plan` reports
+ each as [`SETTING_MOVED`](/docs/reference/errors#SETTING_MOVED), a warning.
+ In a flow's or a flow table's `with`, its fix writes the intent and removes
+ the setting; anywhere else, such as the standard, write the intent by hand.
+- **The `databricks:` section.** The platform's `workspace`, `mode`, `run_as`,
+ `compute`, `permissions`, `manage` and `grants` now go under `databricks:`,
+ and `notifications.on_failure` becomes an email alert in `alerts`. At the
+ top they still work, with a
+ [`PLATFORM_FIELD_MOVED`](/docs/reference/errors#PLATFORM_FIELD_MOVED)
+ warning whose fix moves each; one written in both places with different
+ values is [`BAD_SETTING`](/docs/reference/errors#BAD_SETTING). See
+ [the `databricks:` section](/docs/reference/packs/databricks#platform).
+- **What each pack applies.** A value of intent the table's pack does not
+ apply is [`NOT_APPLIED`](/docs/reference/errors#NOT_APPLIED), and with the
+ standard's `features: ga`, one that needs a feature in preview is
+ [`PREVIEW_FEATURE`](/docs/reference/errors#PREVIEW_FEATURE). Projects
+ written before 2.3 declare no intent, so they get neither.
+- **Published standards.** `ingestron init --kit` starts a project from one,
+ `ingestron standard install` moves a project to another version, and
+ `ingestron standard list` lists them; in VS Code, **New project** offers
+ the published standards for the platform you choose. Hover a `REPLACE_ME` to see where to find its
+ value.
+- **Quality and cleansing.** A layer can say what it does with its rows,
+ `does`; a table can cleanse its columns, keep one row per key, convert time
+ zones, and keep the rows a `drop` check leaves out; and a transform with one
+ input can leave out `code`. See
+ [What a layer does](/docs/concepts/layers#what-a-layer-does),
+ [Check data quality](/docs/how-to/add-quality-checks) and
+ [Cleanse data in the right layer](/docs/how-to/cleanse-data). All of it is
+ optional.
+- **DuckDB and Synapse check rows where a layer says so.** They run a table's
+ checks only in a layer whose `does` lists `check`, and count them, each as
+ `warn`, in a `raw` one. In a layer without `does` their files are what they
+ were, and each contract rule they do not run is a new warning,
+ [`NOT_APPLIED`](/docs/reference/errors#NOT_APPLIED), with the hint to add
+ `check` to the layer's `does`. Expect these on the first plan with 2.3; see
+ [Not applied here](/docs/concepts/not-applied).
+- **Table checks on DuckDB and Synapse need nothing turned on.** Where a layer
+ checks, they count table checks and `unique` in their own run, whatever
+ `manage.checks` says. Only Databricks needs `databricks.manage.checks`.
+- **New codes for old warnings.**
+ - `CONTRACT_FIELD_UNUSED` is no longer reported. A contract field the pack
+ that keeps a table does not apply yet is `NOT_APPLIED` instead, judged for
+ each table against that pack, not against every pack of the standard.
+ - `QUALITY_RULE_NOT_CHECKED` for a check an engine cannot run, such as one
+ on a table that ends at the landed files, is `NOT_APPLIED`. It stays for a
+ rule no engine can check as written, such as a `custom` rule.
+ - A rule that counts over all rows while `databricks.manage.checks` is off
+ was `QUALITY_RULE_NOT_CHECKED`; it is
+ [`CAPABILITY_OFF`](/docs/reference/errors#CAPABILITY_OFF), with the fix
+ that turns checks on.
+ - A flow's `expectations` on a table that Lakeflow Connect writes were an
+ error; each is `NOT_APPLIED`, and so is each required column there.
+
+ A script that looks for an old code should look for the new one.
+
+- **Accepted warnings.** A warning you have decided to live with can be
+ accepted in `platform.yaml`, with the reason; see
+ [Accept a warning](/docs/how-to/accept-a-warning). An entry for
+ `CONTRACT_FIELD_UNUSED` is [`ACCEPT_CODE`](/docs/reference/errors#ACCEPT_CODE),
+ an error: accept `NOT_APPLIED` instead.
+- **Grants, masks and row filters.** A contract's `roles`, and a layer's
+ `access`, can be granted on each table; the standard's `governance` can
+ make grants exact, mask columns by their classification, and filter rows.
+ Databricks applies them after each refresh, with `databricks.manage.grants`,
+ `masks` and `filters`; Synapse grants each view, with
+ `synapse.manage.grants`. `principals` in `platform.yaml` says who each role
+ is. Each switch is off by default, so the files do not change until you
+ turn one on. See
+ [Grant access, mask columns and filter rows](/docs/how-to/grant-access).
+- **New warnings for roles.** A project whose contracts name `roles` gets a
+ warning for each role on the first plan with 2.3, and the same files:
+ - on Databricks or Synapse,
+ [`CAPABILITY_OFF`](/docs/reference/errors#CAPABILITY_OFF), whose fix
+ turns on `databricks.manage.grants` or `synapse.manage.grants`;
+ - on DuckDB, [`NOT_APPLIED`](/docs/reference/errors#NOT_APPLIED): DuckDB
+ has no access control.
+
+ Masks and row filters, which only a standard written for 2.3 can give,
+ warn for each classified column they mask and each filter:
+ `CAPABILITY_OFF` on Databricks, and `NOT_APPLIED` on Synapse and DuckDB,
+ which cannot mask or filter.
+ Turn the capability on, or accept the warning with
+ `about: { feature: grants }` (or `write`, `exact`, `masks` or `filters`)
+ and the reason; see [Accept a gap](/docs/how-to/grant-access#accept-a-gap).
+ Either way, the passport's Who may use it says which roles are granted
+ and which are not enforced, and why.
+
+- **A role's `access` is checked where it is granted.** A role whose
+ `access` is neither `read` nor `write` is
+ [`ROLE_ACCESS`](/docs/reference/errors#ROLE_ACCESS), an error, only on a
+ table where its grant would be applied. A project that does not turn grants
+ on gets no new error.
+- **Tag keys Unity Catalog refuses.** With `databricks.manage.tags` on, a
+ contract tag key, or the classification's tag key, holding
+ `* < > % & ? \`, a control character, or a space at either end is now
+ [`PACK_CHECK`](/docs/reference/errors#PACK_CHECK), an error, as `. , - = / :`
+ already were. Unity Catalog refused such a key when the task ran; now
+ `plan` says so first. Rename the tag.
+- **A Databricks `run_as` user name with a dot.** `prerequisites/.sql`
+ wrote a `run_as` `user_name` such as `first.last@example.com` as two names,
+ `` `first`.`last@example.com` ``, which is not the user. It is now one
+ name, `` `first.last@example.com` ``, so that file changes for such a
+ project. A service principal's application ID has no dot, and is written as
+ before.
+- **A classified column that already has the tag key.** On Databricks, a
+ column whose contract `tags` already hold the classification's tag key, and
+ that has a `classification` too, was tagged with the key twice. The apply
+ task now sets it once, from the `classification`, so that task changes for
+ such a project.
+- **`ingestron request` lists each pack's switches.** A Synapse switch is read
+ from the platform's `synapse:` section and listed as
+ `synapse.manage.grants`; Databricks' list is unchanged. A name holding a
+ backtick is written as code with two backticks around it.
+
+To move a project to the new places, plan with 2.3.0, apply each fix (in
+VS Code, **Fix all** applies one kind at once), and plan again: the plan should
+list no change to the generated files, other than the passports. Then read
+what it lists under `Not applied on this platform`, and decide for each
+whether to give the layer a duty, change the definitions or accept it.
+
+For code that calls the packages:
+
+- `@ingestron/definitions` 1.13 adds intent: `intentPaths`, `intentUseOf` and
+ `intentCoverageGaps`; a pack's `intent`, and a method's `intent`,
+ `intentDefaults` and `aliases`; each table's `intent` and `intentOrigins`;
+ a pack's `platformSection` and `platformSectionReference`; and `find` and
+ `suggest` on each `FieldInfo`.
+- It also adds quality and cleansing: an expectation's `rule` (`RowRule`); a
+ table's `does`, `cleanse` (`TableCleanse` and `ColumnCleanse`), `timezone`,
+ `quality`, `qualityOrigins`, `quarantine` and `notApplied`; a coverage
+ entry's `onlyWhere` and `zonesNotApplied`; a diagnostic's `subject`; a
+ result's `accepted` and `acceptAtBuild`, and `acceptsWarning`; and
+ `severities` and `origin` in each code's help (`CodeHelp`).
+ `CONTRACT_FIELD_UNUSED` stays in `DefinitionCode`, no longer reported.
+- It also adds access: each table's `access` (`TableAccess`, with
+ `AccessGrant`, `ColumnMask`, `RowFilter` and `FilterMatch`), a serve step's
+ `notApplied`, the platform's `principals`, the `grants`, `masks` and
+ `filters` capabilities, `classificationTag` and `defaultClassificationTag`,
+ and the codes `ROLE_ACCESS`, `MASK_TYPE`, `FILTER_MATCH`, `FILTER_MULTIPLE`,
+ `MASK_NEEDS_RUN_AS`, `MASK_FUNCTIONS`, `POLICY_LIMIT` and
+ `POLICY_PRINCIPALS`.
+- `@ingestron/config` adds `matchesImport`, the one rule `imports` and an
+ accepted warning's `file` follow, and `@ingestron/core`'s `Report` gains
+ `accepted`.
+- `@ingestron/core` 1.1 adds published standards: `installedKits`,
+ `readKit`, `parseKitSpec` and `kitSchema`.
+- `@ingestron/authoring` 2.3 adds `ProjectSnapshot.replaceMeHelp`, what a
+ `REPLACE_ME` stands for, and lists a table's intent in its hover.
+
+Release 2.3.1 changes only what the command and the editor say; the generated
+files are the same:
+
+- When an error stops `ingestron request`, the request lists each error, as
+ `ingestron plan` writes it, after its note. See
+ [When the definitions have problems](/docs/how-to/ask-the-platform-team#when-the-definitions-have-problems).
+- `plan` and `apply` count the places still `REPLACE_ME`:
+ `8 places are still REPLACE_ME; …`. `init` counts the README's rows, one
+ per value, so a value written for each environment is one row there and
+ one place per environment here.
+- In VS Code 2.2.1, completion offers a method's settings that are now intent,
+ such as `mode` and `sequence_by`, after the others, each with the intent
+ that replaces it.
+
+## Ingestron 2.4 [#upgrade-to-2-4]
+
+2.4 generates the same files as 2.3 for a project that changes nothing. It
+adds [source metadata](/docs/how-to/describe-a-source): what each source table
+looks like, imported from a file its owners send, so a table that copies the
+source as it is can do without a contract. It also adds
+[a guide for coding assistants](/docs/tools/assistants). What you may notice:
+
+- **A flow table's `contract` is optional.** A table without one takes its
+ columns and key from the source metadata. A table that needs one, because
+ its layer cleanses, checks or quarantines, the platform manages its access
+ or it has its own `expectations`, is
+ [`CONTRACT_NEEDED`](/docs/reference/errors#CONTRACT_NEEDED); one with
+ neither a contract nor metadata is
+ [`TABLE_NO_COLUMNS`](/docs/reference/errors#TABLE_NO_COLUMNS). The editor
+ no longer marks a missing `contract` as a schema error. Every table of a
+ 2.3 project names a contract, so none of these is new to it.
+- **`init` asks one more question,** in a terminal, last: which coding
+ assistants to write files for. An empty answer writes none, and
+ `--assistant` or `--yes` answers it.
+- **Words.** The packs' setting descriptions and some messages say "the
+ table's columns" where they said "the contract's", since the columns can now
+ come from the source metadata. Databricks' `lakeflow_connect.columns` value
+ `contract` is now described as "Only the table's columns."
+- **VS Code 2.3** adds **Request source metadata**, **Import source
+ metadata…** and **Add assistant files**, imports a file saved under
+ `sources/`, and completes recipes and column names; see
+ [VS Code](/docs/tools/vscode#source-metadata).
+
+For code that calls the packages:
+
+- `@ingestron/definitions` 1.14 adds the `SourceMetadata` document and its
+ types, `Contract.origin`, and the `sourceTables` option of resolving, which
+ lists each flow table with its entry in the metadata.
+- `@ingestron/core` 1.2 adds the recipes (`sourceRecipes`,
+ `installedSourceRecipes`, `readSourceRecipe` and `sourceOrigin`), the
+ request (`sourceRequest`), the import (`importSource`) and drafting
+ (`draftContracts`). It reads the recipe catalogue from
+ `@ingestron/standards` 1.1.
+- `@ingestron/authoring` 2.4 adds `importSource`, `sourceRequest` and
+ `sourceFileAt` to `ProjectSnapshot`, and `replaceFrom` to a `Suggestion`.
+
+## Ingestron 2.5 [#upgrade-to-2-5]
+
+2.5 lets a project start from the standard and the platform, before any
+source. For a project that changes nothing, the only difference in the files
+it generates is how the SQL is laid out. What you may notice:
+
+- **Start from the standard and the platform.**
+ `ingestron init --platform `
+ creates a project with a standard and target settings and no source. `--standard ` uses your
+ team's standard, and `--kit --empty` a published one without
+ its example. A new project imports only the files it has; the first
+ connection, flow or contract adds its folder. See
+ [Create a project](/docs/how-to/start-from-a-template).
+- **Values built from values.** A value may refer to another, such as
+ `${values.landing_catalog}`; see
+ [YAML references](/docs/concepts/yaml-references#project-values-and-environments).
+ A loop of references is
+ [`VALUE_CYCLE`](/docs/reference/errors#VALUE_CYCLE), which replaces
+ `VALUES_SELF_REFERENCE`.
+- **A flow's own triggers** on Azure Data Factory and Synapse:
+ `ingest.triggers` (schedules and tumbling windows) and `ingest.parameters`,
+ as the standard allows; see
+ [Run a flow on its own triggers](/docs/adf/run-a-flow-on-its-own-triggers).
+ A flow that writes none keeps its default trigger. Trigger time zones use
+ the Windows names from the Unicode CLDR.
+- **Synapse workspace pipelines** copy Azure SQL tables as full snapshots.
+- **Column types from the source metadata.** With Synapse as the standard's
+ first pack, `money` becomes `DECIMAL(19,4)` and `smallmoney`
+ `DECIMAL(10,4)`, and a contract drafted from the metadata keeps such a type
+ as `physicalType`; see [Column types](/docs/synapse#column-types). The
+ SQL Server recipe in `@ingestron/standards` 1.2 gives `money` and
+ `smallmoney` their precision and scale.
+- **Generated SQL** lists each column on its own line on Databricks and
+ DuckDB, and in Synapse's checks. The first `plan` after upgrading shows
+ those files as updated; what they do is unchanged.
+- **Databricks:** a project with no flow yet plans without a problem, and
+ `init --platform databricks` writes `workspace` under `databricks:`.
+- **VS Code 2.4:** **New project** asks where the project runs, which
+ standard and which assistant files; **Add connection** and **Add flow** are
+ in the Project view; each table has one **Generated files** lens, and
+ **Draft contract** beside it when the table has imported metadata; requests
+ open in named, read-only tabs. See [VS Code](/docs/tools/vscode).
+- **Known gap:** in an output folder with no earlier build, a Data Factory
+ tumbling-window trigger whose start is in the past is refused.
+
+For code that calls the packages:
+
+- `@ingestron/config` 1.8: `VALUE_CYCLE` replaces `VALUES_SELF_REFERENCE` in
+ `DiagnosticCode`.
+- `@ingestron/definitions` 1.15 adds a flow's `triggers` and `parameters`,
+ and a source table's `types`: the columns whose metadata type differs from
+ their logical type's.
+- `@ingestron/core` 1.3 adds the platform starters, and reads the recipe
+ catalogue from `@ingestron/standards` 1.2.
+- `@ingestron/authoring` 2.5 adds `replaceTo` to a `Suggestion`: completion
+ inside `${…}` replaces the whole name.
diff --git a/content/docs/(reference)/reference/templates.mdx b/content/docs/(reference)/reference/templates.mdx
index fae4e91..bd69034 100644
--- a/content/docs/(reference)/reference/templates.mdx
+++ b/content/docs/(reference)/reference/templates.mdx
@@ -22,7 +22,7 @@ value still `REPLACE_ME`; `--set` writes them as the project is created:
ingestron init my-project --template retail --yes
```
-In VS Code, **Ingestron: New project** then **From a starter** lists the same
+In VS Code, **Ingestron: New project** then **Try a sample project** lists the same
starters by title and name. Either way, the result is ordinary definitions you
edit; [Create a project](/docs/how-to/start-from-a-template#start-from-a-starter)
covers folders and Git URLs, and `--standard` and `--set`. To start from a
@@ -82,7 +82,7 @@ files from these downloads;
[Create a project](/docs/how-to/start-from-a-template#from-a-downloaded-example)
says how to try one. To learn the workflow, follow
[Run your first project on your laptop](/docs/start/first-project), which uses
-the `duckdb` starter, or [Azure Data Factory](/docs/platforms/adf), which uses
+the `duckdb` starter, or [Azure Data Factory](/docs/adf), which uses
`connection-first-adf`.
diff --git a/content/docs/(reference)/reference/versions.mdx b/content/docs/(reference)/reference/versions.mdx
index 71c4d37..0e2789b 100644
--- a/content/docs/(reference)/reference/versions.mdx
+++ b/content/docs/(reference)/reference/versions.mdx
@@ -29,5 +29,5 @@ Connector maturity, such as Public Preview or Beta, is shown on each
The [deployment templates](/docs/how-to/deploy-through-environments) pin the
tools their stages run: `@microsoft/azure-data-factory-utilities` 1.0.3, the
Databricks CLI v1.23.0, and sqlcmd (Go) v1.10.0. Their check stage installs
-`ingestron@2.4.0`, and each GitHub action is pinned to a commit, with its
+`ingestron@2.5.0`, and each GitHub action is pinned to a commit, with its
version beside it.
diff --git a/content/docs/(reference)/tools/assistants.mdx b/content/docs/(reference)/tools/assistants.mdx
index cc875bc..c23a8fd 100644
--- a/content/docs/(reference)/tools/assistants.mdx
+++ b/content/docs/(reference)/tools/assistants.mdx
@@ -59,13 +59,21 @@ AGENTS.md: written
-**Ingestron: New project** asks, after what to start from and before the
-folder, **New project: AI assistant files (optional)**. Tick any of **GitHub
+**Ingestron: New project** asks, after where the project runs and which
+standard or sample, and before the folder, **New project: AI assistant files
+(optional)**. Tick any of **GitHub
Copilot**, **Claude Code**, **Codex**, **Cursor** and **Another tool that
reads AGENTS.md**, or press Enter with nothing ticked for none.
+
+
For a project you already have, run **Ingestron: Add assistant files** from
-the Command Palette, or right-click the project's folder, or one of its files,
+the Command Palette, or right-click the project root or `project.yaml`
in the Explorer. Tick one or more. The Ingestron output panel lists each file,
and a notification offers **Open AGENTS.md**. If a file could not be written,
a warning says so, and the output panel says why.
diff --git a/content/docs/(reference)/tools/vscode.mdx b/content/docs/(reference)/tools/vscode.mdx
index 01bc9d2..a0aab79 100644
--- a/content/docs/(reference)/tools/vscode.mdx
+++ b/content/docs/(reference)/tools/vscode.mdx
@@ -26,8 +26,8 @@ in the workspace.
## Get started
@@ -37,6 +37,20 @@ project to generated files and a request for your platform team. Each step
is ticked when you do it, such as when the workspace has a flow, or after
**Ingestron: Apply** runs.
+Run **Ingestron: New project**: it asks where the project runs, a platform or
+a sample to try, then which standard, a new one for that platform, one of its
+published standards, or your team's file, then which assistant files to
+write, if any. Review the target settings and
+environments first. Then use
+**Add connection** and **Add flow** in the Project panel or Command Palette.
+Connection creation offers metadata request/import without creating a flow;
+flow creation selects an existing connection and source objects. Existing
+standards and definitions are preserved. A new project imports only the
+files it has, such as `standard.yaml` and `platform.yaml`; the first connection,
+flow or contract adds its folder to `imports` and says so. These actions save
+each file they change, unless it had unsaved changes: that file stays unsaved,
+and the message names it. See [Create a project](/docs/how-to/start-from-a-template).
+
## Problems as you type
Definition problems appear in the Problems panel at their file and line,
@@ -67,7 +81,7 @@ Ingestron cannot read is a warning (`ENV_FILE_LINE`) at that line. See
@@ -116,10 +130,17 @@ CLI prints the first fix under the problem, as `Fix: …`.
+
+
When the same problem has a fix in several places, the menu also offers
**Fix all N … problems**, which applies each one's first fix in one change; an
edit several of them share, such as turning on `databricks.manage.checks`, is
@@ -158,7 +179,8 @@ the standard or platform reaches every table that uses the method.
Completion offers what fits where you type: fields from the document's
schema, allowed values and examples, the ids of connections and contracts,
table names in `inputs`, the forms a reference
-takes after `${` and value names after `${values.`, the names a naming pattern can use inside `{{ }}`, the methods
+takes after `${` and value names after `${values.` (in `project.yaml`, every value but
+the one you are writing; after `${values.cluster.`, the names inside `cluster`), the names a naming pattern can use inside `{{ }}`, the methods
for `uses`, and, under `with`, the settings of the method that table uses
in the environment in the status bar, with their defaults. This works in a
new file too, before it has ever been valid.
@@ -237,9 +259,12 @@ For the environment in the status bar, the editor shows beside each line what
it resolves to:
- after a flow table or a transform, the table's full name;
-- above it, the methods that build it, with how each was chosen on hover, and
- each file generated for it, which opens that file read-only. A contract
- object shows the same files;
+- above it, **Generated files (N)** opens a picker of the assets generated for
+ that table. Their full paths stay in the picker; hover on the link for the
+ method choices. A contract object shows the same action;
+- a metadata-based flow table also offers **Draft contract**, even when the
+ standard does not require one. It creates an ODCS draft and adds its
+ `contract:` reference as one undoable change. Review the draft before use;
- after a value with `${…}`, the value it resolves to. Secrets and environment
variables are never shown.
@@ -273,6 +298,23 @@ field is, where to find the value, and a suggestion of what it looks like,
such as ``. A value in `project.yaml`'s `values` is explained
by the field that reads it.
+Hover a flow's trigger, anywhere under `ingest.triggers`, to see when it next
+starts in the selected environment, in the trigger's own time zone, such as
+`Next starts in prod (Europe/London): Sun 6 Dec 03:00, Sun 13 Dec 03:00, Sun 20 Dec 03:00.`
+A start outside the standard's hours is not hidden: the Problems panel reports
+it as [`TRIGGER_HOURS`](/docs/reference/errors#TRIGGER_HOURS). See
+[Run a flow on its own triggers](/docs/how-to/run-a-flow-on-its-own-triggers).
+
+Hover a `${values.…}` reference to see its value in the selected environment
+and where it is set: under `values`, or under that environment's `values`. A
+value built from other values also shows how it is written, such as
+`/Volumes/${values.landing_catalog}/finance/files`.
+
+Completion leaves out a method's settings that are now intent, such as `mode`,
+except under the standard's `fixed`. One already written still works, with
+[`SETTING_MOVED`](/docs/reference/errors#SETTING_MOVED); hover it to see the
+intent to write instead, such as `load.mode`.
+
Hover any field or setting to see what it is, what each allowed value means,
its default, an example and where it is usually set; hover a value for that
value's meaning. Each ends with a **Reference** link to its entry in these
@@ -287,8 +329,10 @@ The **Ingestron** tab in the secondary side bar, on the right, opens with a
**Project** section. For a workspace with one project it lists:
- **Environments**, each with a circle, or a check and **selected** on the one
- problems, previews and the platform request show. Click an environment to
- select it, as in the status bar.
+ problems, previews and the platform request show, and its mode,
+ **development** or **production**. Click an environment to select it, as in
+ the status bar.
+- **Add connection** and **Add flow**, as the commands of the same name.
- **Check**, **Plan**, **Apply**, **Compare environments**, **Prepare platform
request**, **Request source metadata**, **Import source metadata…** and
**Show generated files**. Click one to run it for the project. Check, Plan
@@ -307,7 +351,7 @@ one to the file adds it to the list.
@@ -330,7 +374,7 @@ Explorer.
@@ -387,7 +431,7 @@ it.
environment in the status bar: who to ask, the query to run or the call to
make, filled in for your project, and where to save the result. With more
than one connection, choose one, or **All connections**. The request opens
- as an unsaved Markdown document, ready to send; nothing is written to the
+ in a read-only tab, ready to copy and send; nothing is written to the
project.
- **Ingestron: Import source metadata…** reads a file into
`sources/.yaml`. Run it on a project, and choose the file; or
@@ -434,6 +478,11 @@ recipes are shown in the editor only. `ingestron source request` and
[Use your own recipe](/docs/reference/recipes#your-own). While a recipe is broken, no other recipe problem
is shown, and completion offers no recipes.
+A source object absent from the imported catalogue is still reported when an
+unrelated document is unfinished, such as an empty contract file. The diagnostic
+checks imported metadata rather than making a live database request. Generation
+remains blocked until the document errors are fixed.
+
## Plan, apply and check
**Ingestron: Plan**, **Ingestron: Apply** and **Ingestron: Check** run the
@@ -445,12 +494,15 @@ output panel.
You can start them from where you are working, as well as from the Command
Palette. Each acts on the project you started it from:
-- **Right-click** a project folder, or a YAML file in a project, in the
- Explorer for **Ingestron: Check**, **Plan**, **Apply**, **Compare
- environments**, **Prepare platform request**, **Request source metadata**
- and **Add assistant files**. A project folder also has **Import source
- metadata…**, as does any file that is not one of a project's definitions.
- The files in `build/` do not have the menu.
+- **Right-click** the project root or `project.yaml` in the Explorer for
+ **Check**, **Plan**, **Apply**, environment comparison, platform requests and
+ project setup actions. Other YAML files have actions for their purpose:
+ connection files offer **Request source metadata**; flow files offer
+ **Show generated files** and **Draft contract from metadata**.
+ The `connections/` and `flows/` folders offer **Add connection** and **Add flow**;
+ `sources/` offers **Import source metadata…**. Folders containing those document
+ kinds also get the matching actions when you use a custom layout.
+ Contract files and generated YAML do not receive the project action list.
- **Editor title buttons**: with a YAML file of a project, or a preview of
generated files, open, **Plan** and **Apply** are buttons in the editor's
title bar. **Check**, **Compare environments**, **Prepare platform
@@ -461,7 +513,7 @@ Palette. Each acts on the project you started it from:
@@ -473,8 +525,8 @@ Palette. Each acts on the project you started it from:
/>
@@ -490,6 +542,13 @@ for each output folder, such as `databricks (dev, prod): up to date`, with the
files it changed below it, and a line for each folder that is no longer
generated.
+
+
It applies nothing while the definitions have problems in any environment,
and says where to see them: for another environment, such as
`not applied: prod has problems (select prod to see them in the Problems panel)`.
@@ -506,28 +565,28 @@ workspace.
## Start a project
-**Ingestron: New project** starts from a connection, a starter or a published
-standard.
-
-- **From a connection** asks for a source category, connection, network
- reachability and supported route. You can search across all sources or
- within a category.
-- **From a starter** offers the ready-made projects, such as **DuckDB on a
- laptop**, which [runs locally with DuckDB](/docs/platforms/duckdb).
-- **From a published standard** offers the dated standards installed with the
- extension, such as **Local medallion on DuckDB**, and makes the same project
- as `ingestron init --kit`; see
- [Start from a published standard](/docs/how-to/start-from-a-published-standard).
-
-It then offers a list of coding assistants to write files for, which you may
-skip: press Enter with nothing ticked for none. Nothing else is asked. The
-destination folder comes last, and its name is the
-project's id; the environments are `dev` and `prod`, or the starter's or the
-published standard's, and the standard is the route's, the starter's or the
-published one. The new `README.md` opens in a
-preview, and a message says how many values are still `REPLACE_ME`: the
-README lists each with where to find it. When the folder is outside the
-workspace, the message offers to open it or add it to the workspace. See
+**Ingestron: New project** asks three questions, then for the folder.
+
+- **Where will this project run?** Azure Synapse, Databricks, Azure Data
+ Factory (landing) or DuckDB. **Try a sample project** instead lists complete
+ worked projects, such as DuckDB on a laptop.
+- **Which standard should it follow?** **A new standard for** the platform
+ creates a reviewable local standard and target settings, with no source. A
+ published standard for that platform uses its dated standard and target
+ settings, without the worked source example (`init --kit --empty`). **Your
+ team's standard file** keeps your standard YAML.
+- **Assistant files**, optional: the coding assistants to write files for;
+ press Enter with nothing ticked for none. See
+ [Work with an AI assistant](/docs/tools/assistants).
+
+Then choose the folder and **Create the project here**: the folder's name is
+the project's id. The environments are `dev` and `prod`, or the sample's or
+the published standard's, and the standard is the platform's, the sample's or
+the published one. The new `README.md` opens in a preview, and a message says
+how many values are still `REPLACE_ME`: the README lists each with where to
+find it. When the folder is outside the workspace, the message offers to open
+it or add it to the workspace; when it is in the workspace, the message offers
+**Add connection**, which lists only the sources the platform supports. See
[Create a project](/docs/how-to/start-from-a-template).
**Ingestron: Add assistant files** writes the same files into a project you
@@ -564,3 +623,9 @@ your environments. The Ingestron extension does, as
## Every command
+
+## Review source metadata and generated files
+
+Field completions insert a YAML delimiter and offer values. On an imported source object, use Go to Definition to inspect its captured metadata; the Reference panel and hover show its columns and capture provenance. If an object is misspelt, the diagnostic distinguishes it from an uncaptured source and offers nearby imported names.
+
+The environment rows show each resolved environment’s effective mode. Compare environments before accepting a new review or staging environment. Requests and comparisons open in named, read-only tabs that refresh in place; use Save As to retain evidence. Plan, Apply and Check write and review local files. They do not deploy or run the generated code.
diff --git a/content/docs/(reference)/troubleshooting.mdx b/content/docs/(reference)/troubleshooting.mdx
index 9ae6d0d..a20a277 100644
--- a/content/docs/(reference)/troubleshooting.mdx
+++ b/content/docs/(reference)/troubleshooting.mdx
@@ -1,6 +1,6 @@
---
title: Troubleshooting
-description: Find a problem by what you see, then its cause and fix, for the command, generated files, definitions, environment variables, the editor and each route.
+description: Find a problem by what you see, then its cause and fix, for the command, generated files, definitions, source metadata, environment variables and the editor.
---
Find the message or behaviour you see. Each entry gives the cause and the fix,
@@ -499,329 +499,15 @@ the environment in the status bar makes no difference.
[Upgrade Ingestron and its packs](/docs/how-to/upgrade) describes, and reload
the window after installing the extension again.
-## Azure Data Factory
+## Your platform [#platforms]
-### A connection is missing a setting
+
+
+
+
-**Symptom:** the plan stops before any file is written:
-
-```text
-flows/postgresql.yaml:15:5 error METHOD_CHECK
- "adf-table" for table "postgresql_records": the connection needs "settings.server" as non-empty text.
-```
-
-**Cause:** the connector needs a setting the connection does not give, or the
-connection gives one the connector does not accept
-(`the connection setting "ssl" is not supported for postgresql`).
-
-**Fix:** add the setting to the connection, or remove the one the message says
-is not supported. Find your connector in the [catalogue](/docs/connectors); its
-page, such as [PostgreSQL](/docs/connectors/postgresql), lists the connection
-fields. See [Azure Data Factory](/docs/platforms/adf).
-
-### A pipeline fails when it runs
-
-**Symptom:** `plan` and `apply` succeed, but a deployed pipeline fails in Data
-Factory before it copies, or while it reads the source or writes the landing
-folder.
-
-**Cause:** Ingestron generates the factory's files. It does not provision the
-runtime, the Key Vault or the storage, and does not test credentials or
-network access. Whatever is missing, such as the runtime, a secret, the
-identity's access to the Key Vault or the landing storage, or the source
-permissions and runtime prerequisites, fails only when the pipeline runs.
-
-**Fix:** run `ingestron request --env dev`, or **Ingestron: Prepare platform
-request** in VS Code. Its **ADF prerequisites** section lists, with this
-project's values, the integration runtime, the Key Vault, the landing folder
-and the source's permissions that must exist. Give the list to the team that
-creates them. See
-[Before you deploy](/docs/platforms/adf-deploy#prerequisites).
-
-### Every plan and check prints a notice about a runtime [#runtime-notice]
-
-**Symptom:** `plan`, `apply` and `check` print a warning before the folders,
-and still succeed:
-
-```text
-adf: warnings
-(project) warning PACK_NOTICE
- Integration runtime `shir-erp` (self-hosted): the deployment writes it by name and type, as Data Factory Studio does; where it creates it, register a node, with network access and any required source drivers, before the copies run.
-```
-
-**Cause:** the factory holds each integration runtime its linked services
-use, and an ARM deployment sets a runtime's whole resource. The
-[`PACK_NOTICE`](/docs/reference/errors#PACK_NOTICE) says what a deployment
-does to it. It is a warning, so it shows on every run, in CI too.
-
-**Fix:** nothing to change in the definitions. Before the first deployment to
-a factory that already has the runtime, run `az deployment group what-if` with
-the template and that environment's parameter file, and describe a linked or
-Azure runtime with the method's `runtime` setting; see
-[Runtime and Key Vault](/docs/platforms/adf#runtime-and-key-vault).
-
-### Development triggers do not run
-
-**Symptom:** the pipelines are in the factory, but a `dev` copy never starts
-on its schedule.
-
-**Cause:** triggers start only in `prod` and `production`
-([platform defaults](/docs/reference/packs/adf#platform-configuration)). The
-trigger file in `factory/trigger/` says `"runtimeState": "Stopped"`, and
-`parameters/prod.json` sets the trigger's state to `"Started"`.
-
-**Fix:** start the trigger in Data Factory when you want development copies to
-run, or set `paused: false` for the copy method in the platform and generate
-again, which starts it in every environment. Check `start` first: a started trigger runs every window since `start`. See [Retry and pause](/docs/platforms/adf-pipelines#retry-and-pause).
-
-## Synapse serverless views
-
-### A view's SQL starts with WITH
-
-**Symptom:**
-
-```text
-transforms/customer_totals.yaml:3:5 error METHOD_CHECK
- "synapse-sql" for table "customer_totals": a view's SQL cannot start with WITH; write the common table expression as a subquery.
-```
-
-**Cause:** a transform's SQL sits inside `CREATE VIEW`, so it cannot open with
-a common table expression.
-
-**Fix:** write the expression as a subquery in `FROM`. A transform's SQL also
-names only the tables listed in `inputs`; the message
-`the SQL names {{ tables.customers }}, which is not in inputs` means one is missing.
-See [What each engine makes of it](/docs/how-to/write-a-transform#what-each-engine-makes-of-it).
-
-### A view reads files outside the data source
-
-**Symptom:**
-
-```text
-flows/erp.yaml:6:14 error METHOD_CHECK
- "synapse-view" for table "customers": abfss://landing@saleslakedev.dfs.core.windows.net/erp/erp.dbo.Customers/erp.dbo.Customers.parquet is not under the data source landing (abfss://other@saleslakedev.dfs.core.windows.net).
-```
-
-**Cause:** the files ADF lands are not under `data_source_location` in the
-platform, so the external data source cannot reach them.
-
-**Fix:** set `data_source_location` to a folder above every landing folder, or
-point the landing folder inside it. See
-[Settings that shape it](/docs/platforms/synapse#settings).
-
-### Running the views fails the first time [#running-viewssql-fails-the-first-time]
-
-**Symptom:** `sqlcmd -i environments/.sql` fails in Synapse for a
-new environment, although `apply` succeeded.
-
-**Cause:** Ingestron writes the views but creates nothing in Synapse. The
-serverless database, a master key, the credential, the external data source and
-the workspace's read access to the storage must exist first.
-
-**Fix:** run `ingestron request --env dev`. Its **Synapse serverless database**,
-**External data source** and **Report users** sections list each statement and
-permission, with this project's names. Ask the team that owns the workspace to
-create them, then run the environment's script again.
-
-### sqlcmd cannot find views.sql
-
-**Symptom:** `sqlcmd -i environments/.sql` stops at once, saying
-it cannot open `views.sql`, or Synapse Studio reports an error at the first
-`$(…)` in `views.sql`.
-
-**Cause:** sqlcmd reads the `:r views.sql` line from the folder it starts in,
-not from the script's folder. `views.sql` holds sqlcmd variables, which only
-sqlcmd replaces, so Synapse Studio cannot run it.
-
-**Fix:** run sqlcmd from `build/synapse`, with the environment's script. See
-[Deploy](/docs/platforms/synapse#deploy).
-
-## DuckDB
-
-### Append would miss later loads
-
-**Symptom:**
-
-```text
-flows/erp.yaml:6:13 error METHOD_CHECK
- "duckdb-table" for table "invoices": append reads each file once, but "adf-copy" replaces its file each run, so later loads would be missed; use replace or upsert.
-```
-
-**Cause:** `append` reads each file name once. A copy that replaces its file
-under the same name every load would never be read again. For the same reason,
-a file you re-export under its old name is skipped.
-
-**Fix:** keep the table with `history: none` (the method's `replace`) or
-`history: latest` (`upsert`) instead of `mode: append`, or give each load its
-own file, such as with `increments`. To reload everything, delete `ingestron.duckdb` and run
-again. See [Keep a table current](/docs/how-to/keep-a-table-current#on-duckdb).
-
-### Upsert needs a primary key
-
-**Symptom:**
-
-```text
-flows/erp.yaml:6:13 error METHOD_CHECK
- "duckdb-table" for table "invoices": upsert needs the contract's primaryKey.
-```
-
-**Cause:** `history: latest`, the method's `upsert`, replaces rows with the
-same key, and the contract names no key. `sequence` has to be a column of the
-contract too.
-
-**Fix:** mark the key column `primaryKey: true` in the contract. See
-[Keep a table current](/docs/how-to/keep-a-table-current).
-
-### A run keeps going after an error
-
-**Symptom:** a DuckDB error is printed, but the run carries on, and some tables
-and Parquet files exist although another table failed.
-
-**Cause:** as the generated `README.md` says, without `-bail` DuckDB carries on
-after an error and commits what it built. With `-bail`, a run that fails while
-building publishes nothing.
-
-**Fix:** run it as the generated `README.md` says, from the project folder:
-
-```sh
-duckdb -bail ingestron.duckdb -c ".read build/duckdb/dev/run.sql"
-```
-
-### A check stops the run [#a-check-stops-the-run]
-
-**Symptom:** the run prints its checks, each with its failing rows, then stops
-with:
-
-```text
-Invalid Input Error: The check "" failed for rows of
, so nothing is published.
-```
-
-or, for a check over the whole table,
-`The table check "" failed on
, so nothing is published.`
-
-**Cause:** a check whose action is `fail` found rows that fail it. The run
-stops before it inserts them: with `-bail`, nothing is published and no table
-changes.
-
-**Check:** the table the run printed just before the error lists each check,
-its action and its `failing_rows`. Query the files the table reads for the
-rows the check describes.
-
-**Fix:** correct the data and run again. If such rows should be kept and
-counted, or left out, change the check's action to `warn` or `drop`; see
-[Check data quality](/docs/how-to/add-quality-checks).
-
-### Publishing fails after the tables are built
-
-**Symptom:** the run reaches the step that publishes and fails there, or no
-`.
.parquet` file appears.
-
-**Cause:** `run.sql` publishes each table to the `published` folder in
-`platform.yaml`, and that folder must exist.
-
-**Fix:** create the folder, such as `published/`, or correct the setting, and
-run again. Running again is safe.
-
-### Paths mix local and Azure
-
-**Symptom:** the environment's `README.md` begins:
-
-```text
-This environment reads or writes both local paths and paths in Azure storage,
-and local paths cannot run in Azure: a job there cannot see them. Give it only
-paths in Azure storage, or only local paths. `run.sql` has no Azure sign-in of
-its own.
-```
-
-and `ingestron request` adds `give this environment only paths in Azure storage`.
-
-**Cause:** one path in the environment is local, such as `./published`, and
-another is in Azure. A job in Azure cannot see a local path, and `run.sql`
-has no Azure sign-in of its own, so a laptop cannot read the Azure ones.
-
-**Fix:** give the environment only Azure paths, or only local ones. To try a
-cloud project on a laptop, use an environment with `runs-on: duckdb`. See
-[Run a cloud project locally](/docs/platforms/duckdb#run-a-cloud-project-locally).
-
-### The job after the copies fails or times out
-
-**Symptom:** the parent pipeline `pl__every__` fails
-in Data Factory with one of:
-
-```text
-The job after the copies ended Failed.
-The job after the copies did not finish within 01:00:00, so it was stopped.
-The job after the copies did not finish within 01:00:00 and could not be stopped:
-```
-
-The status can be another word than `Failed`, and the time is the
-`after_copy` timeout.
-
-**Cause:** the Container Apps job ended without succeeding, or ran past the
-timeout (one hour by default). The pipeline reads the run's status every 30
-seconds. The last message means ADF could not stop the run: the factory's
-identity may lack the **Container Apps Jobs Operator** role on the job.
-
-**Fix:** open the job's run in Azure for the error. The job retries a failed run
-when its retry limit is 1, as `ingestron request` asks, and running again is
-safe because each run rebuilds everything.
-Raise `after_copy.timeout` for a larger load. `ingestron request --env dev`
-lists the job and the access it needs. See
-[Run and operate](/docs/platforms/duckdb-in-azure#operate).
-
-## Databricks
-
-### A change mode needs a key or a sequence column
-
-**Symptom:**
-
-```text
-flows/crm.yaml:9:5 error METHOD_CHECK
- "streaming-table" for table "accounts": mode upsert needs "sequence_by".
-```
-
-or `mode upsert needs primary key columns in the contract`.
-
-**Cause:** `history: latest` or `full`, which the method writes as
-`mode: upsert` or `mode: history`, keeps a table current by key, in the order
-of a column.
-
-**Fix:** set `sequence` on the flow or the table to the column that orders a
-key's changes, and mark the key columns `primaryKey: true` in the contract.
-See [Keep a table current](/docs/how-to/keep-a-table-current).
-
-### Databricks rejects the bundle
-
-**Symptom:** `databricks bundle validate` or `deploy` fails.
-
-**Check:** read Databricks' own message; Ingestron does not produce it. If a
-schema is missing, the schemas are not created yet: ask the team that owns
-them, or see
-[Let the bundle create the schemas](/docs/platforms/databricks#create-the-schemas).
-A Lakeflow Connect pipeline refers to a Unity Catalog connection, which
-Ingestron does not create.
-
-**Fix:** when the platform sets `databricks.manage.schemas`, an administrator runs the
-generated `prerequisites/.sql` once before the first deploy: it grants
-`USE CATALOG` and `CREATE SCHEMA` on each catalog to the identity the pipelines
-run as, and the same must be given to the identity that deploys when it
-differs. Run `ingestron request` for everything else the platform team
-provides.
-
-### The refresh job does not run in development
-
-**Symptom:** the job and the pipelines are deployed, and nothing refreshes on
-its schedule in `dev`.
-
-**Cause:** schedules run only in `prod` and `production`
-([platform defaults](/docs/reference/packs/databricks#platform-configuration)).
-`dev` is in development mode, which pauses every schedule, and another
-environment's target in `databricks.yml` sets
-`presets: { trigger_pause_status: PAUSED }`.
-
-**Fix:** unpause it in Databricks, or set `paused: false` in the platform's
-`schedule` and generate again: the job then says `pause_status: UNPAUSED` in
-every environment. See
-[Refresh on a schedule](/docs/platforms/databricks#schedule).
-
-Every code is explained in [Errors](/docs/reference/errors).
+Problems with what each platform runs are on its own troubleshooting page:
+[Azure Synapse](/docs/synapse/troubleshooting),
+[Databricks](/docs/databricks/troubleshooting),
+[Azure Data Factory](/docs/adf/troubleshooting) and
+[DuckDB](/docs/duckdb/troubleshooting).
diff --git a/content/docs/adf-databricks/(guides)/deploy.mdx b/content/docs/adf-databricks/(guides)/deploy.mdx
new file mode 100644
index 0000000..7230b7a
--- /dev/null
+++ b/content/docs/adf-databricks/(guides)/deploy.mdx
@@ -0,0 +1,50 @@
+---
+title: Deploy both
+description: Deploy each environment's factory, then its Databricks bundle target, by hand or from CI.
+---
+
+Ingestron writes the factory and the bundle and deploys neither. Deploy each
+environment's factory first, then the bundle's target: the pipelines read only
+what the factory lands.
+
+## Before you deploy [#prerequisites]
+
+`ingestron request --env dev` lists what both need, with this project's
+values: the integration runtime, Key Vault and landing storage for the factory,
+then the workspace, catalog and identities for the bundle. Each platform's
+page says more about its part:
+[the factory](/docs/adf/deploy#prerequisites) and
+[the bundle](/docs/databricks/deploy#prerequisites).
+
+## Deploy [#deploy]
+
+1. Run `ingestron apply`.
+2. Build the factory's ARM template once and deploy it with the environment's
+ parameter file, as in
+ [Build the template and deploy it](/docs/adf/deploy#build-and-deploy).
+3. Deploy the bundle's target for the environment from `build/databricks`, as
+ in [Deploy the Databricks bundle](/docs/databricks/deploy#deploy):
+
+ ```sh
+ cd build/databricks
+ databricks bundle deploy -t dev
+ ```
+
+The factory's triggers and the bundle's job are stopped outside production.
+`paused: true` in the platform's `schedule` stops both in production too; to
+stop only the copies, see
+[Retry and pause](/docs/adf/pipelines#retry-and-pause).
+
+### From CI [#deploy-from-ci]
+
+Each environment's stage in the CI templates deploys the factory, then the
+bundle, in that order. Each platform's page gives its steps:
+[the factory](/docs/adf/deploy#deploy-from-ci) and
+[the bundle](/docs/databricks/deploy#deploy-from-ci).
+
+## Next
+
+- [Load with Azure Data Factory](/docs/adf-databricks/load-with-adf): what each
+ part writes, and the settings that shape it.
+- [Run the factory](/docs/adf/deploy#operate) and
+ [run the refresh](/docs/databricks/deploy#operate).
diff --git a/content/docs/adf-databricks/(guides)/load-with-adf.mdx b/content/docs/adf-databricks/(guides)/load-with-adf.mdx
new file mode 100644
index 0000000..40145a4
--- /dev/null
+++ b/content/docs/adf-databricks/(guides)/load-with-adf.mdx
@@ -0,0 +1,83 @@
+---
+title: Load with Azure Data Factory
+description: Let Azure Data Factory copy a source Databricks cannot reach into landing storage, and read the landed files into Databricks tables.
+---
+
+When Databricks cannot reach a source, such as a database on a private
+network, Azure Data Factory copies it into ADLS Gen2 landing storage through
+an integration runtime. The Databricks pack then writes the pipelines that read
+the landed files into tables, on their own schedule.
+
+
+
+The factory is described on [Azure Data Factory](/docs/adf): its runtime,
+Key Vault, schedules and pipelines are the same whatever reads the files. This
+page covers what Databricks adds.
+
+## Create it [#create-it]
+
+
+
+
+From a parent folder, start from the starter that copies a private SQL Server:
+
+```sh
+ingestron init erp --template private-sql-server --yes
+cd erp
+```
+
+For your own source, name its connector and the route instead:
+
+```sh
+ingestron init sales --connector postgresql --connectivity private --via adf-databricks --yes
+```
+
+
+
+
+Run **Ingestron: New project**, choose **Try a sample project**, then
+**Private SQL Server through Azure Data Factory**, then the folder. For your
+own source, create the project in a terminal with the command on the CLI tab,
+then open its folder in VS Code.
+
+
+
+
+
+
+The [connector catalogue](/docs/~/connectors) lists every source and whether ADF
+reads it. The project with its values written is a download:
+
+
+
+## What Databricks writes [#what-you-get]
+
+Each landed table becomes a streaming table that reads each new file once,
+with Auto Loader, from the landing path ADF writes to:
+
+
+
+A native table copy lands the whole selection each window, so with the default
+store repeated copies add repeated rows. Choose `history: latest` or `full`
+to keep the current row per key or every version; see
+[Keep a table current](/docs/databricks/keep-a-table-current). A SQL-family
+copy can land only the rows changed in each window, `load: incremental`; see
+[Choose the source and its tables](/docs/adf#choose-the-source-and-its-tables).
+
+## Deploy [#deploy]
+
+Deploy the factory first, as in [Deploy the factory](/docs/adf/deploy#deploy),
+then the bundle, as in [Deploy the Databricks bundle](/docs/databricks/deploy#deploy).
+The request lists what both need: see
+[Ask the platform team](/docs/databricks/ask-the-platform-team).
+
+## Next
+
+- [Shape ADF pipelines](/docs/adf/pipelines): schedules, retries and landing paths.
+- [Keep a table current](/docs/databricks/keep-a-table-current): current rows or history from repeated copies.
+- [Deploy through environments](/docs/databricks/deploy-through-environments).
diff --git a/content/docs/adf-databricks/(guides)/meta.json b/content/docs/adf-databricks/(guides)/meta.json
new file mode 100644
index 0000000..42d1f58
--- /dev/null
+++ b/content/docs/adf-databricks/(guides)/meta.json
@@ -0,0 +1,6 @@
+{
+ "title": "Guides",
+ "description": "Loading the landed files into Databricks, and deploying both.",
+ "root": true,
+ "pages": ["load-with-adf", "run-a-flow-on-its-own-triggers", "deploy"]
+}
diff --git a/content/docs/adf-databricks/(guides)/run-a-flow-on-its-own-triggers.mdx b/content/docs/adf-databricks/(guides)/run-a-flow-on-its-own-triggers.mdx
new file mode 100644
index 0000000..96be9cf
--- /dev/null
+++ b/content/docs/adf-databricks/(guides)/run-a-flow-on-its-own-triggers.mdx
@@ -0,0 +1,6 @@
+---
+title: Run a flow on its own triggers
+description: Write an hourly window and a weekly full reload for one flow, inside the hours your standard allows, on Azure Data Factory.
+---
+
+../../../shared/guides/run-a-flow-on-its-own-triggers.mdx
diff --git a/content/docs/adf-databricks/(start)/first-project.mdx b/content/docs/adf-databricks/(start)/first-project.mdx
new file mode 100644
index 0000000..acd9a2e
--- /dev/null
+++ b/content/docs/adf-databricks/(start)/first-project.mdx
@@ -0,0 +1,6 @@
+---
+title: Your first Data Factory into Databricks project
+description: Create a project in which Data Factory copies one table from a private SQL Server and a Databricks pipeline reads it, on your computer. Nothing is deployed.
+---
+
+../../../shared/start/first-project.mdx
diff --git a/content/docs/adf-databricks/(start)/index.mdx b/content/docs/adf-databricks/(start)/index.mdx
new file mode 100644
index 0000000..5293b7c
--- /dev/null
+++ b/content/docs/adf-databricks/(start)/index.mdx
@@ -0,0 +1,58 @@
+---
+title: Data Factory into Databricks
+description: Azure Data Factory copies each source into landing storage, and Databricks pipelines read the landed files into tables, from one project.
+---
+
+Use this combination when Databricks cannot reach a source, such as a database
+on a private network, or when a factory already copies your sources. Azure
+Data Factory copies each table into ADLS Gen2 landing storage through an
+integration runtime, on its schedule. Databricks pipelines then read each new
+landed file into a streaming table, on theirs.
+
+
+
+To try it on your computer first, follow
+[Your first project](/docs/adf-databricks/first-project).
+
+## What each part writes [#what-you-get]
+
+One project, with one standard that lists both packs, writes two sets of
+files:
+
+- **The factory**, in `build/adf`: the integration runtime, linked services
+ for the Key Vault, the source and the landing storage, datasets for the
+ source and the landing storage, the copy pipeline and its trigger, and a
+ parameter file for each environment.
+- **The bundle**, in `build/databricks`: a pipeline with a streaming table for
+ each landed table, a job that refreshes it, and a target for each
+ environment.
+
+Each table lands in its own folder under the landing path, and the table's SQL
+reads that folder with Auto Loader, so each new file is read once:
+
+
+
+## What it looks like in Databricks [#on-the-platform]
+
+
+
+## The rest is each platform's
+
+Everything else works as it does on each platform alone:
+
+- **Data Factory:** [the factory's settings](/docs/adf#runtime-and-key-vault),
+ [schedules, retries and landing paths](/docs/adf/pipelines), and
+ [a flow's own triggers](/docs/adf/run-a-flow-on-its-own-triggers).
+- **Databricks:** [keep a table current](/docs/databricks/keep-a-table-current),
+ [write a transform](/docs/databricks/write-a-transform),
+ [check data quality](/docs/databricks/add-quality-checks) and
+ [grant access](/docs/databricks/grant-access).
+- **Standards:** the standard lists both packs, so each pack's rules apply to
+ its files; see [Databricks standards](/docs/databricks/configure-a-standard)
+ and [Data Factory standards](/docs/adf/configure-a-standard).
diff --git a/content/docs/adf-databricks/(start)/meta.json b/content/docs/adf-databricks/(start)/meta.json
new file mode 100644
index 0000000..d7b9c06
--- /dev/null
+++ b/content/docs/adf-databricks/(start)/meta.json
@@ -0,0 +1,6 @@
+{
+ "title": "Get started",
+ "description": "What Data Factory and Databricks each write, and a first project.",
+ "root": true,
+ "pages": ["index", "first-project"]
+}
diff --git a/content/docs/adf-databricks/connectors/adls-gen2.mdx b/content/docs/adf-databricks/connectors/adls-gen2.mdx
new file mode 100644
index 0000000..63ac4cf
--- /dev/null
+++ b/content/docs/adf-databricks/connectors/adls-gen2.mdx
@@ -0,0 +1,6 @@
+---
+title: Azure Data Lake Storage Gen2
+description: Azure Data Factory lands Azure Data Lake Storage Gen2 files in storage, and Databricks reads the landed files into tables.
+---
+
+../../../shared/connectors/adls-gen2.mdx
diff --git a/content/docs/adf-databricks/connectors/amazon-rds-oracle.mdx b/content/docs/adf-databricks/connectors/amazon-rds-oracle.mdx
new file mode 100644
index 0000000..4be3a92
--- /dev/null
+++ b/content/docs/adf-databricks/connectors/amazon-rds-oracle.mdx
@@ -0,0 +1,6 @@
+---
+title: Amazon RDS for Oracle
+description: Azure Data Factory lands Amazon RDS for Oracle tables in storage, and Databricks reads the landed files into tables.
+---
+
+../../../shared/connectors/amazon-rds-oracle.mdx
diff --git a/content/docs/adf-databricks/connectors/amazon-rds-sqlserver.mdx b/content/docs/adf-databricks/connectors/amazon-rds-sqlserver.mdx
new file mode 100644
index 0000000..3bd4e32
--- /dev/null
+++ b/content/docs/adf-databricks/connectors/amazon-rds-sqlserver.mdx
@@ -0,0 +1,6 @@
+---
+title: Amazon RDS for SQL Server
+description: Azure Data Factory lands a full table or a time window of Amazon RDS for SQL Server in storage, and Databricks reads the landed files into tables.
+---
+
+../../../shared/connectors/amazon-rds-sqlserver.mdx
diff --git a/content/docs/adf-databricks/connectors/amazon-redshift.mdx b/content/docs/adf-databricks/connectors/amazon-redshift.mdx
new file mode 100644
index 0000000..c7270ab
--- /dev/null
+++ b/content/docs/adf-databricks/connectors/amazon-redshift.mdx
@@ -0,0 +1,6 @@
+---
+title: Amazon Redshift
+description: Azure Data Factory lands Amazon Redshift tables in storage, and Databricks reads the landed files into tables.
+---
+
+../../../shared/connectors/amazon-redshift.mdx
diff --git a/content/docs/adf-databricks/connectors/amazon-s3.mdx b/content/docs/adf-databricks/connectors/amazon-s3.mdx
new file mode 100644
index 0000000..340d0ca
--- /dev/null
+++ b/content/docs/adf-databricks/connectors/amazon-s3.mdx
@@ -0,0 +1,6 @@
+---
+title: Amazon S3
+description: Azure Data Factory lands Amazon S3 files in storage, and Databricks reads the landed files into tables.
+---
+
+../../../shared/connectors/amazon-s3.mdx
diff --git a/content/docs/adf-databricks/connectors/azure-blob.mdx b/content/docs/adf-databricks/connectors/azure-blob.mdx
new file mode 100644
index 0000000..84578e3
--- /dev/null
+++ b/content/docs/adf-databricks/connectors/azure-blob.mdx
@@ -0,0 +1,6 @@
+---
+title: Azure Blob Storage
+description: Azure Data Factory lands Azure Blob Storage files in storage, and Databricks reads the landed files into tables.
+---
+
+../../../shared/connectors/azure-blob.mdx
diff --git a/content/docs/adf-databricks/connectors/azure-cosmos-db-mongodb.mdx b/content/docs/adf-databricks/connectors/azure-cosmos-db-mongodb.mdx
new file mode 100644
index 0000000..f807901
--- /dev/null
+++ b/content/docs/adf-databricks/connectors/azure-cosmos-db-mongodb.mdx
@@ -0,0 +1,6 @@
+---
+title: Azure Cosmos DB for MongoDB
+description: Azure Data Factory lands Azure Cosmos DB for MongoDB tables in storage, and Databricks reads the landed files into tables.
+---
+
+../../../shared/connectors/azure-cosmos-db-mongodb.mdx
diff --git a/content/docs/adf-databricks/connectors/azure-cosmos-db-nosql.mdx b/content/docs/adf-databricks/connectors/azure-cosmos-db-nosql.mdx
new file mode 100644
index 0000000..00dd634
--- /dev/null
+++ b/content/docs/adf-databricks/connectors/azure-cosmos-db-nosql.mdx
@@ -0,0 +1,6 @@
+---
+title: Azure Cosmos DB for NoSQL
+description: Azure Data Factory lands Azure Cosmos DB for NoSQL tables in storage, and Databricks reads the landed files into tables.
+---
+
+../../../shared/connectors/azure-cosmos-db-nosql.mdx
diff --git a/content/docs/adf-databricks/connectors/azure-cosmos-postgresql.mdx b/content/docs/adf-databricks/connectors/azure-cosmos-postgresql.mdx
new file mode 100644
index 0000000..ca544fe
--- /dev/null
+++ b/content/docs/adf-databricks/connectors/azure-cosmos-postgresql.mdx
@@ -0,0 +1,6 @@
+---
+title: Azure Cosmos DB for PostgreSQL
+description: Azure Data Factory lands Azure Cosmos DB for PostgreSQL tables in storage, and Databricks reads the landed files into tables.
+---
+
+../../../shared/connectors/azure-cosmos-postgresql.mdx
diff --git a/content/docs/adf-databricks/connectors/azure-data-explorer.mdx b/content/docs/adf-databricks/connectors/azure-data-explorer.mdx
new file mode 100644
index 0000000..08fd356
--- /dev/null
+++ b/content/docs/adf-databricks/connectors/azure-data-explorer.mdx
@@ -0,0 +1,6 @@
+---
+title: Azure Data Explorer
+description: Azure Data Factory lands Azure Data Explorer tables in storage, and Databricks reads the landed files into tables.
+---
+
+../../../shared/connectors/azure-data-explorer.mdx
diff --git a/content/docs/adf-databricks/connectors/azure-databricks-delta.mdx b/content/docs/adf-databricks/connectors/azure-databricks-delta.mdx
new file mode 100644
index 0000000..853f256
--- /dev/null
+++ b/content/docs/adf-databricks/connectors/azure-databricks-delta.mdx
@@ -0,0 +1,6 @@
+---
+title: Azure Databricks Delta Lake
+description: Azure Data Factory lands Azure Databricks Delta Lake tables in storage, and Databricks reads the landed files into tables.
+---
+
+../../../shared/connectors/azure-databricks-delta.mdx
diff --git a/content/docs/adf-databricks/connectors/azure-files.mdx b/content/docs/adf-databricks/connectors/azure-files.mdx
new file mode 100644
index 0000000..18c8dee
--- /dev/null
+++ b/content/docs/adf-databricks/connectors/azure-files.mdx
@@ -0,0 +1,6 @@
+---
+title: Azure Files
+description: Azure Data Factory lands the files in storage, and Databricks reads the landed files into tables.
+---
+
+../../../shared/connectors/azure-files.mdx
diff --git a/content/docs/adf-databricks/connectors/azure-mysql.mdx b/content/docs/adf-databricks/connectors/azure-mysql.mdx
new file mode 100644
index 0000000..e88879f
--- /dev/null
+++ b/content/docs/adf-databricks/connectors/azure-mysql.mdx
@@ -0,0 +1,6 @@
+---
+title: Azure Database for MySQL
+description: Azure Data Factory lands Azure Database for MySQL tables in storage, and Databricks reads the landed files into tables.
+---
+
+../../../shared/connectors/azure-mysql.mdx
diff --git a/content/docs/adf-databricks/connectors/azure-postgresql.mdx b/content/docs/adf-databricks/connectors/azure-postgresql.mdx
new file mode 100644
index 0000000..52be5ea
--- /dev/null
+++ b/content/docs/adf-databricks/connectors/azure-postgresql.mdx
@@ -0,0 +1,6 @@
+---
+title: Azure Database for PostgreSQL
+description: Azure Data Factory lands Azure Database for PostgreSQL tables in storage, and Databricks reads the landed files into tables.
+---
+
+../../../shared/connectors/azure-postgresql.mdx
diff --git a/content/docs/adf-databricks/connectors/azure-sql-database.mdx b/content/docs/adf-databricks/connectors/azure-sql-database.mdx
new file mode 100644
index 0000000..0dcaae7
--- /dev/null
+++ b/content/docs/adf-databricks/connectors/azure-sql-database.mdx
@@ -0,0 +1,6 @@
+---
+title: Azure SQL Database
+description: Azure Data Factory lands a full table or a time window of Azure SQL Database in storage, and Databricks reads the landed files into tables.
+---
+
+../../../shared/connectors/azure-sql-database.mdx
diff --git a/content/docs/adf-databricks/connectors/azure-sql-managed-instance.mdx b/content/docs/adf-databricks/connectors/azure-sql-managed-instance.mdx
new file mode 100644
index 0000000..8a8dda6
--- /dev/null
+++ b/content/docs/adf-databricks/connectors/azure-sql-managed-instance.mdx
@@ -0,0 +1,6 @@
+---
+title: Azure SQL Managed Instance
+description: Azure Data Factory lands a full table or a time window of Azure SQL Managed Instance in storage, and Databricks reads the landed files into tables.
+---
+
+../../../shared/connectors/azure-sql-managed-instance.mdx
diff --git a/content/docs/adf-databricks/connectors/azure-synapse.mdx b/content/docs/adf-databricks/connectors/azure-synapse.mdx
new file mode 100644
index 0000000..4da6c61
--- /dev/null
+++ b/content/docs/adf-databricks/connectors/azure-synapse.mdx
@@ -0,0 +1,6 @@
+---
+title: Azure Synapse Analytics
+description: Azure Data Factory lands a full table or a time window of Azure Synapse Analytics in storage, and Databricks reads the landed files into tables.
+---
+
+../../../shared/connectors/azure-synapse.mdx
diff --git a/content/docs/adf-databricks/connectors/azure-table-storage.mdx b/content/docs/adf-databricks/connectors/azure-table-storage.mdx
new file mode 100644
index 0000000..dfe962f
--- /dev/null
+++ b/content/docs/adf-databricks/connectors/azure-table-storage.mdx
@@ -0,0 +1,6 @@
+---
+title: Azure Table Storage
+description: Azure Data Factory lands Azure Table Storage tables in storage, and Databricks reads the landed files into tables.
+---
+
+../../../shared/connectors/azure-table-storage.mdx
diff --git a/content/docs/adf-databricks/connectors/cassandra.mdx b/content/docs/adf-databricks/connectors/cassandra.mdx
new file mode 100644
index 0000000..109354a
--- /dev/null
+++ b/content/docs/adf-databricks/connectors/cassandra.mdx
@@ -0,0 +1,6 @@
+---
+title: Cassandra
+description: Azure Data Factory lands Cassandra tables in storage, and Databricks reads the landed files into tables.
+---
+
+../../../shared/connectors/cassandra.mdx
diff --git a/content/docs/adf-databricks/connectors/dataverse.mdx b/content/docs/adf-databricks/connectors/dataverse.mdx
new file mode 100644
index 0000000..3318f74
--- /dev/null
+++ b/content/docs/adf-databricks/connectors/dataverse.mdx
@@ -0,0 +1,6 @@
+---
+title: Dataverse
+description: Azure Data Factory lands Dataverse tables in storage, and Databricks reads the landed files into tables.
+---
+
+../../../shared/connectors/dataverse.mdx
diff --git a/content/docs/adf-databricks/connectors/db2.mdx b/content/docs/adf-databricks/connectors/db2.mdx
new file mode 100644
index 0000000..3a43020
--- /dev/null
+++ b/content/docs/adf-databricks/connectors/db2.mdx
@@ -0,0 +1,6 @@
+---
+title: DB2
+description: Azure Data Factory lands DB2 tables in storage, and Databricks reads the landed files into tables.
+---
+
+../../../shared/connectors/db2.mdx
diff --git a/content/docs/adf-databricks/connectors/dynamics-365.mdx b/content/docs/adf-databricks/connectors/dynamics-365.mdx
new file mode 100644
index 0000000..ba5bdb9
--- /dev/null
+++ b/content/docs/adf-databricks/connectors/dynamics-365.mdx
@@ -0,0 +1,6 @@
+---
+title: Dynamics 365
+description: Azure Data Factory lands Dynamics 365 tables in storage, and Databricks reads the landed files into tables.
+---
+
+../../../shared/connectors/dynamics-365.mdx
diff --git a/content/docs/adf-databricks/connectors/dynamics-ax.mdx b/content/docs/adf-databricks/connectors/dynamics-ax.mdx
new file mode 100644
index 0000000..3bddd06
--- /dev/null
+++ b/content/docs/adf-databricks/connectors/dynamics-ax.mdx
@@ -0,0 +1,6 @@
+---
+title: Dynamics AX
+description: Azure Data Factory lands Dynamics AX tables in storage, and Databricks reads the landed files into tables.
+---
+
+../../../shared/connectors/dynamics-ax.mdx
diff --git a/content/docs/adf-databricks/connectors/dynamics-crm.mdx b/content/docs/adf-databricks/connectors/dynamics-crm.mdx
new file mode 100644
index 0000000..444e518
--- /dev/null
+++ b/content/docs/adf-databricks/connectors/dynamics-crm.mdx
@@ -0,0 +1,6 @@
+---
+title: Dynamics CRM
+description: Azure Data Factory lands Dynamics CRM tables in storage, and Databricks reads the landed files into tables.
+---
+
+../../../shared/connectors/dynamics-crm.mdx
diff --git a/content/docs/adf-databricks/connectors/fabric-lakehouse.mdx b/content/docs/adf-databricks/connectors/fabric-lakehouse.mdx
new file mode 100644
index 0000000..7f5080a
--- /dev/null
+++ b/content/docs/adf-databricks/connectors/fabric-lakehouse.mdx
@@ -0,0 +1,6 @@
+---
+title: Microsoft Fabric Lakehouse
+description: Azure Data Factory lands Microsoft Fabric Lakehouse tables in storage, and Databricks reads the landed files into tables.
+---
+
+../../../shared/connectors/fabric-lakehouse.mdx
diff --git a/content/docs/adf-databricks/connectors/fabric-warehouse.mdx b/content/docs/adf-databricks/connectors/fabric-warehouse.mdx
new file mode 100644
index 0000000..6a13ae9
--- /dev/null
+++ b/content/docs/adf-databricks/connectors/fabric-warehouse.mdx
@@ -0,0 +1,6 @@
+---
+title: Microsoft Fabric Warehouse
+description: Azure Data Factory lands Microsoft Fabric Warehouse tables in storage, and Databricks reads the landed files into tables.
+---
+
+../../../shared/connectors/fabric-warehouse.mdx
diff --git a/content/docs/adf-databricks/connectors/files.mdx b/content/docs/adf-databricks/connectors/files.mdx
new file mode 100644
index 0000000..6565d99
--- /dev/null
+++ b/content/docs/adf-databricks/connectors/files.mdx
@@ -0,0 +1,6 @@
+---
+title: Files
+description: Azure Data Factory lands the files in storage, and Databricks reads the landed files into tables.
+---
+
+../../../shared/connectors/files.mdx
diff --git a/content/docs/adf-databricks/connectors/ftp.mdx b/content/docs/adf-databricks/connectors/ftp.mdx
new file mode 100644
index 0000000..bdd018e
--- /dev/null
+++ b/content/docs/adf-databricks/connectors/ftp.mdx
@@ -0,0 +1,6 @@
+---
+title: FTP
+description: Azure Data Factory lands FTP files in storage, and Databricks reads the landed files into tables.
+---
+
+../../../shared/connectors/ftp.mdx
diff --git a/content/docs/adf-databricks/connectors/google-ads.mdx b/content/docs/adf-databricks/connectors/google-ads.mdx
new file mode 100644
index 0000000..5b932d2
--- /dev/null
+++ b/content/docs/adf-databricks/connectors/google-ads.mdx
@@ -0,0 +1,6 @@
+---
+title: Google Ads
+description: Azure Data Factory lands Google Ads tables in storage, and Databricks reads the landed files into tables.
+---
+
+../../../shared/connectors/google-ads.mdx
diff --git a/content/docs/adf-databricks/connectors/google-bigquery.mdx b/content/docs/adf-databricks/connectors/google-bigquery.mdx
new file mode 100644
index 0000000..4e226fa
--- /dev/null
+++ b/content/docs/adf-databricks/connectors/google-bigquery.mdx
@@ -0,0 +1,6 @@
+---
+title: Google BigQuery
+description: Azure Data Factory lands Google BigQuery tables in storage, and Databricks reads the landed files into tables.
+---
+
+../../../shared/connectors/google-bigquery.mdx
diff --git a/content/docs/adf-databricks/connectors/google-cloud-storage.mdx b/content/docs/adf-databricks/connectors/google-cloud-storage.mdx
new file mode 100644
index 0000000..b26c152
--- /dev/null
+++ b/content/docs/adf-databricks/connectors/google-cloud-storage.mdx
@@ -0,0 +1,6 @@
+---
+title: Google Cloud Storage
+description: Azure Data Factory lands Google Cloud Storage files in storage, and Databricks reads the landed files into tables.
+---
+
+../../../shared/connectors/google-cloud-storage.mdx
diff --git a/content/docs/adf-databricks/connectors/greenplum.mdx b/content/docs/adf-databricks/connectors/greenplum.mdx
new file mode 100644
index 0000000..4a2299a
--- /dev/null
+++ b/content/docs/adf-databricks/connectors/greenplum.mdx
@@ -0,0 +1,6 @@
+---
+title: Greenplum
+description: Azure Data Factory lands Greenplum tables in storage, and Databricks reads the landed files into tables.
+---
+
+../../../shared/connectors/greenplum.mdx
diff --git a/content/docs/adf-databricks/connectors/hdfs.mdx b/content/docs/adf-databricks/connectors/hdfs.mdx
new file mode 100644
index 0000000..8d1c92d
--- /dev/null
+++ b/content/docs/adf-databricks/connectors/hdfs.mdx
@@ -0,0 +1,6 @@
+---
+title: HDFS
+description: Azure Data Factory lands HDFS files in storage, and Databricks reads the landed files into tables.
+---
+
+../../../shared/connectors/hdfs.mdx
diff --git a/content/docs/adf-databricks/connectors/hive.mdx b/content/docs/adf-databricks/connectors/hive.mdx
new file mode 100644
index 0000000..4fdcb26
--- /dev/null
+++ b/content/docs/adf-databricks/connectors/hive.mdx
@@ -0,0 +1,6 @@
+---
+title: Hive
+description: Azure Data Factory lands Hive tables in storage, and Databricks reads the landed files into tables.
+---
+
+../../../shared/connectors/hive.mdx
diff --git a/content/docs/adf-databricks/connectors/http.mdx b/content/docs/adf-databricks/connectors/http.mdx
new file mode 100644
index 0000000..b706b00
--- /dev/null
+++ b/content/docs/adf-databricks/connectors/http.mdx
@@ -0,0 +1,6 @@
+---
+title: HTTP
+description: Azure Data Factory lands HTTP files in storage, and Databricks reads the landed files into tables.
+---
+
+../../../shared/connectors/http.mdx
diff --git a/content/docs/adf-databricks/connectors/hubspot.mdx b/content/docs/adf-databricks/connectors/hubspot.mdx
new file mode 100644
index 0000000..a3deae9
--- /dev/null
+++ b/content/docs/adf-databricks/connectors/hubspot.mdx
@@ -0,0 +1,6 @@
+---
+title: HubSpot
+description: Azure Data Factory lands HubSpot tables in storage, and Databricks reads the landed files into tables.
+---
+
+../../../shared/connectors/hubspot.mdx
diff --git a/content/docs/adf-databricks/connectors/impala.mdx b/content/docs/adf-databricks/connectors/impala.mdx
new file mode 100644
index 0000000..c1ca125
--- /dev/null
+++ b/content/docs/adf-databricks/connectors/impala.mdx
@@ -0,0 +1,6 @@
+---
+title: Apache Impala
+description: Azure Data Factory lands Apache Impala tables in storage, and Databricks reads the landed files into tables.
+---
+
+../../../shared/connectors/impala.mdx
diff --git a/content/docs/adf-databricks/connectors/index.mdx b/content/docs/adf-databricks/connectors/index.mdx
new file mode 100644
index 0000000..f76c109
--- /dev/null
+++ b/content/docs/adf-databricks/connectors/index.mdx
@@ -0,0 +1,6 @@
+---
+title: Connectors
+description: The sources Data Factory into Databricks can load, and how each one is read.
+---
+
+../../../shared/connectors/index.mdx
diff --git a/content/docs/adf-databricks/connectors/informix.mdx b/content/docs/adf-databricks/connectors/informix.mdx
new file mode 100644
index 0000000..7bc5a8d
--- /dev/null
+++ b/content/docs/adf-databricks/connectors/informix.mdx
@@ -0,0 +1,6 @@
+---
+title: Informix
+description: Azure Data Factory lands Informix tables in storage, and Databricks reads the landed files into tables.
+---
+
+../../../shared/connectors/informix.mdx
diff --git a/content/docs/adf-databricks/connectors/jira.mdx b/content/docs/adf-databricks/connectors/jira.mdx
new file mode 100644
index 0000000..2351bfc
--- /dev/null
+++ b/content/docs/adf-databricks/connectors/jira.mdx
@@ -0,0 +1,6 @@
+---
+title: Jira
+description: Azure Data Factory lands Jira tables in storage, and Databricks reads the landed files into tables.
+---
+
+../../../shared/connectors/jira.mdx
diff --git a/content/docs/adf-databricks/connectors/mariadb.mdx b/content/docs/adf-databricks/connectors/mariadb.mdx
new file mode 100644
index 0000000..1fc8c7e
--- /dev/null
+++ b/content/docs/adf-databricks/connectors/mariadb.mdx
@@ -0,0 +1,6 @@
+---
+title: MariaDB
+description: Azure Data Factory lands MariaDB tables in storage, and Databricks reads the landed files into tables.
+---
+
+../../../shared/connectors/mariadb.mdx
diff --git a/content/docs/adf-databricks/connectors/meta.json b/content/docs/adf-databricks/connectors/meta.json
new file mode 100644
index 0000000..7c77468
--- /dev/null
+++ b/content/docs/adf-databricks/connectors/meta.json
@@ -0,0 +1,89 @@
+{
+ "title": "Connectors",
+ "description": "The sources Data Factory into Databricks can load, and how each one is read.",
+ "root": true,
+ "pages": [
+ "index",
+ "[APIs and protocols](/docs/adf-databricks/connectors#apis-and-protocols)",
+ "http",
+ "odata",
+ "odbc",
+ "rest",
+ "web-table",
+ "[Business applications](/docs/adf-databricks/connectors#business-applications)",
+ "dataverse",
+ "dynamics-365",
+ "dynamics-ax",
+ "dynamics-crm",
+ "google-ads",
+ "hubspot",
+ "jira",
+ "microsoft-365",
+ "quickbooks",
+ "salesforce",
+ "salesforce-service-cloud",
+ "servicenow",
+ "sharepoint-online-list",
+ "shopify",
+ "square",
+ "xero",
+ "[Databases and analytics](/docs/adf-databricks/connectors#databases-and-analytics)",
+ "amazon-rds-oracle",
+ "amazon-rds-sqlserver",
+ "amazon-redshift",
+ "impala",
+ "azure-cosmos-db-mongodb",
+ "azure-cosmos-db-nosql",
+ "azure-cosmos-postgresql",
+ "azure-data-explorer",
+ "azure-mysql",
+ "azure-postgresql",
+ "azure-databricks-delta",
+ "azure-sql-database",
+ "azure-sql-managed-instance",
+ "azure-synapse",
+ "azure-table-storage",
+ "cassandra",
+ "db2",
+ "google-bigquery",
+ "greenplum",
+ "hive",
+ "informix",
+ "mariadb",
+ "microsoft-access",
+ "fabric-lakehouse",
+ "fabric-warehouse",
+ "mongodb",
+ "mongodb-atlas",
+ "mysql",
+ "netezza",
+ "oracle",
+ "postgresql",
+ "presto",
+ "snowflake",
+ "spark",
+ "sqlserver",
+ "sybase",
+ "teradata",
+ "vertica",
+ "[Files and storage](/docs/adf-databricks/connectors#files-and-storage)",
+ "amazon-s3",
+ "s3-compatible",
+ "azure-blob",
+ "adls-gen2",
+ "azure-files",
+ "files",
+ "ftp",
+ "google-cloud-storage",
+ "hdfs",
+ "oracle-cloud-storage",
+ "sftp",
+ "[SAP](/docs/adf-databricks/connectors#sap)",
+ "sap-bw-open-hub",
+ "sap-bw-mdx",
+ "sap-c4c",
+ "sap-ecc",
+ "sap-hana",
+ "sap-table"
+ ]
+}
diff --git a/content/docs/adf-databricks/connectors/microsoft-365.mdx b/content/docs/adf-databricks/connectors/microsoft-365.mdx
new file mode 100644
index 0000000..017fa69
--- /dev/null
+++ b/content/docs/adf-databricks/connectors/microsoft-365.mdx
@@ -0,0 +1,6 @@
+---
+title: Microsoft 365
+description: Azure Data Factory lands Microsoft 365 tables in storage, and Databricks reads the landed files into tables.
+---
+
+../../../shared/connectors/microsoft-365.mdx
diff --git a/content/docs/adf-databricks/connectors/microsoft-access.mdx b/content/docs/adf-databricks/connectors/microsoft-access.mdx
new file mode 100644
index 0000000..9d834b4
--- /dev/null
+++ b/content/docs/adf-databricks/connectors/microsoft-access.mdx
@@ -0,0 +1,6 @@
+---
+title: Microsoft Access
+description: Azure Data Factory lands Microsoft Access tables in storage, and Databricks reads the landed files into tables.
+---
+
+../../../shared/connectors/microsoft-access.mdx
diff --git a/content/docs/adf-databricks/connectors/mongodb-atlas.mdx b/content/docs/adf-databricks/connectors/mongodb-atlas.mdx
new file mode 100644
index 0000000..8fb251d
--- /dev/null
+++ b/content/docs/adf-databricks/connectors/mongodb-atlas.mdx
@@ -0,0 +1,6 @@
+---
+title: MongoDB Atlas
+description: Azure Data Factory lands MongoDB Atlas tables in storage, and Databricks reads the landed files into tables.
+---
+
+../../../shared/connectors/mongodb-atlas.mdx
diff --git a/content/docs/adf-databricks/connectors/mongodb.mdx b/content/docs/adf-databricks/connectors/mongodb.mdx
new file mode 100644
index 0000000..1b68c7d
--- /dev/null
+++ b/content/docs/adf-databricks/connectors/mongodb.mdx
@@ -0,0 +1,6 @@
+---
+title: MongoDB
+description: Azure Data Factory lands MongoDB tables in storage, and Databricks reads the landed files into tables.
+---
+
+../../../shared/connectors/mongodb.mdx
diff --git a/content/docs/adf-databricks/connectors/mysql.mdx b/content/docs/adf-databricks/connectors/mysql.mdx
new file mode 100644
index 0000000..fe79e94
--- /dev/null
+++ b/content/docs/adf-databricks/connectors/mysql.mdx
@@ -0,0 +1,6 @@
+---
+title: MySQL
+description: Azure Data Factory lands MySQL tables in storage, and Databricks reads the landed files into tables.
+---
+
+../../../shared/connectors/mysql.mdx
diff --git a/content/docs/adf-databricks/connectors/netezza.mdx b/content/docs/adf-databricks/connectors/netezza.mdx
new file mode 100644
index 0000000..cb92130
--- /dev/null
+++ b/content/docs/adf-databricks/connectors/netezza.mdx
@@ -0,0 +1,6 @@
+---
+title: Netezza
+description: Azure Data Factory lands Netezza tables in storage, and Databricks reads the landed files into tables.
+---
+
+../../../shared/connectors/netezza.mdx
diff --git a/content/docs/adf-databricks/connectors/odata.mdx b/content/docs/adf-databricks/connectors/odata.mdx
new file mode 100644
index 0000000..3e63dcb
--- /dev/null
+++ b/content/docs/adf-databricks/connectors/odata.mdx
@@ -0,0 +1,6 @@
+---
+title: OData
+description: Azure Data Factory lands OData tables in storage, and Databricks reads the landed files into tables.
+---
+
+../../../shared/connectors/odata.mdx
diff --git a/content/docs/adf-databricks/connectors/odbc.mdx b/content/docs/adf-databricks/connectors/odbc.mdx
new file mode 100644
index 0000000..e4e51df
--- /dev/null
+++ b/content/docs/adf-databricks/connectors/odbc.mdx
@@ -0,0 +1,6 @@
+---
+title: ODBC
+description: Azure Data Factory lands ODBC tables in storage, and Databricks reads the landed files into tables.
+---
+
+../../../shared/connectors/odbc.mdx
diff --git a/content/docs/adf-databricks/connectors/oracle-cloud-storage.mdx b/content/docs/adf-databricks/connectors/oracle-cloud-storage.mdx
new file mode 100644
index 0000000..c040cde
--- /dev/null
+++ b/content/docs/adf-databricks/connectors/oracle-cloud-storage.mdx
@@ -0,0 +1,6 @@
+---
+title: Oracle Cloud Storage
+description: Azure Data Factory lands Oracle Cloud Storage files in storage, and Databricks reads the landed files into tables.
+---
+
+../../../shared/connectors/oracle-cloud-storage.mdx
diff --git a/content/docs/adf-databricks/connectors/oracle.mdx b/content/docs/adf-databricks/connectors/oracle.mdx
new file mode 100644
index 0000000..51be958
--- /dev/null
+++ b/content/docs/adf-databricks/connectors/oracle.mdx
@@ -0,0 +1,6 @@
+---
+title: Oracle
+description: Azure Data Factory lands Oracle tables in storage, and Databricks reads the landed files into tables.
+---
+
+../../../shared/connectors/oracle.mdx
diff --git a/content/docs/adf-databricks/connectors/postgresql.mdx b/content/docs/adf-databricks/connectors/postgresql.mdx
new file mode 100644
index 0000000..5408eb2
--- /dev/null
+++ b/content/docs/adf-databricks/connectors/postgresql.mdx
@@ -0,0 +1,6 @@
+---
+title: PostgreSQL
+description: Azure Data Factory lands PostgreSQL tables in storage, and Databricks reads the landed files into tables.
+---
+
+../../../shared/connectors/postgresql.mdx
diff --git a/content/docs/adf-databricks/connectors/presto.mdx b/content/docs/adf-databricks/connectors/presto.mdx
new file mode 100644
index 0000000..b378f12
--- /dev/null
+++ b/content/docs/adf-databricks/connectors/presto.mdx
@@ -0,0 +1,6 @@
+---
+title: Presto
+description: Azure Data Factory lands Presto tables in storage, and Databricks reads the landed files into tables.
+---
+
+../../../shared/connectors/presto.mdx
diff --git a/content/docs/adf-databricks/connectors/quickbooks.mdx b/content/docs/adf-databricks/connectors/quickbooks.mdx
new file mode 100644
index 0000000..b5c44b2
--- /dev/null
+++ b/content/docs/adf-databricks/connectors/quickbooks.mdx
@@ -0,0 +1,6 @@
+---
+title: QuickBooks Online
+description: Azure Data Factory lands QuickBooks Online tables in storage, and Databricks reads the landed files into tables.
+---
+
+../../../shared/connectors/quickbooks.mdx
diff --git a/content/docs/adf-databricks/connectors/rest.mdx b/content/docs/adf-databricks/connectors/rest.mdx
new file mode 100644
index 0000000..d8f2f5f
--- /dev/null
+++ b/content/docs/adf-databricks/connectors/rest.mdx
@@ -0,0 +1,6 @@
+---
+title: REST
+description: Azure Data Factory lands REST tables in storage, and Databricks reads the landed files into tables.
+---
+
+../../../shared/connectors/rest.mdx
diff --git a/content/docs/adf-databricks/connectors/s3-compatible.mdx b/content/docs/adf-databricks/connectors/s3-compatible.mdx
new file mode 100644
index 0000000..f9b02b4
--- /dev/null
+++ b/content/docs/adf-databricks/connectors/s3-compatible.mdx
@@ -0,0 +1,6 @@
+---
+title: Amazon S3 Compatible Storage
+description: Azure Data Factory lands Amazon S3 Compatible Storage files in storage, and Databricks reads the landed files into tables.
+---
+
+../../../shared/connectors/s3-compatible.mdx
diff --git a/content/docs/adf-databricks/connectors/salesforce-service-cloud.mdx b/content/docs/adf-databricks/connectors/salesforce-service-cloud.mdx
new file mode 100644
index 0000000..b4bd2a2
--- /dev/null
+++ b/content/docs/adf-databricks/connectors/salesforce-service-cloud.mdx
@@ -0,0 +1,6 @@
+---
+title: Salesforce Service Cloud
+description: Azure Data Factory lands Salesforce Service Cloud tables in storage, and Databricks reads the landed files into tables.
+---
+
+../../../shared/connectors/salesforce-service-cloud.mdx
diff --git a/content/docs/adf-databricks/connectors/salesforce.mdx b/content/docs/adf-databricks/connectors/salesforce.mdx
new file mode 100644
index 0000000..91f1767
--- /dev/null
+++ b/content/docs/adf-databricks/connectors/salesforce.mdx
@@ -0,0 +1,6 @@
+---
+title: Salesforce
+description: Azure Data Factory lands Salesforce tables in storage, and Databricks reads the landed files into tables.
+---
+
+../../../shared/connectors/salesforce.mdx
diff --git a/content/docs/adf-databricks/connectors/sap-bw-mdx.mdx b/content/docs/adf-databricks/connectors/sap-bw-mdx.mdx
new file mode 100644
index 0000000..9d220c9
--- /dev/null
+++ b/content/docs/adf-databricks/connectors/sap-bw-mdx.mdx
@@ -0,0 +1,6 @@
+---
+title: SAP Business Warehouse via MDX
+description: Azure Data Factory lands SAP Business Warehouse via MDX tables in storage, and Databricks reads the landed files into tables.
+---
+
+../../../shared/connectors/sap-bw-mdx.mdx
diff --git a/content/docs/adf-databricks/connectors/sap-bw-open-hub.mdx b/content/docs/adf-databricks/connectors/sap-bw-open-hub.mdx
new file mode 100644
index 0000000..bfb8a18
--- /dev/null
+++ b/content/docs/adf-databricks/connectors/sap-bw-open-hub.mdx
@@ -0,0 +1,6 @@
+---
+title: SAP Business Warehouse Open Hub
+description: Azure Data Factory lands SAP Business Warehouse Open Hub tables in storage, and Databricks reads the landed files into tables.
+---
+
+../../../shared/connectors/sap-bw-open-hub.mdx
diff --git a/content/docs/adf-databricks/connectors/sap-c4c.mdx b/content/docs/adf-databricks/connectors/sap-c4c.mdx
new file mode 100644
index 0000000..2d5cc1d
--- /dev/null
+++ b/content/docs/adf-databricks/connectors/sap-c4c.mdx
@@ -0,0 +1,6 @@
+---
+title: SAP Cloud for Customer
+description: Azure Data Factory lands SAP Cloud for Customer tables in storage, and Databricks reads the landed files into tables.
+---
+
+../../../shared/connectors/sap-c4c.mdx
diff --git a/content/docs/adf-databricks/connectors/sap-ecc.mdx b/content/docs/adf-databricks/connectors/sap-ecc.mdx
new file mode 100644
index 0000000..51cbca3
--- /dev/null
+++ b/content/docs/adf-databricks/connectors/sap-ecc.mdx
@@ -0,0 +1,6 @@
+---
+title: SAP ECC
+description: Azure Data Factory lands SAP ECC tables in storage, and Databricks reads the landed files into tables.
+---
+
+../../../shared/connectors/sap-ecc.mdx
diff --git a/content/docs/adf-databricks/connectors/sap-hana.mdx b/content/docs/adf-databricks/connectors/sap-hana.mdx
new file mode 100644
index 0000000..42111ff
--- /dev/null
+++ b/content/docs/adf-databricks/connectors/sap-hana.mdx
@@ -0,0 +1,6 @@
+---
+title: SAP HANA
+description: Azure Data Factory lands SAP HANA tables in storage, and Databricks reads the landed files into tables.
+---
+
+../../../shared/connectors/sap-hana.mdx
diff --git a/content/docs/adf-databricks/connectors/sap-table.mdx b/content/docs/adf-databricks/connectors/sap-table.mdx
new file mode 100644
index 0000000..430dc2e
--- /dev/null
+++ b/content/docs/adf-databricks/connectors/sap-table.mdx
@@ -0,0 +1,6 @@
+---
+title: SAP Table
+description: Azure Data Factory lands SAP Table tables in storage, and Databricks reads the landed files into tables.
+---
+
+../../../shared/connectors/sap-table.mdx
diff --git a/content/docs/adf-databricks/connectors/servicenow.mdx b/content/docs/adf-databricks/connectors/servicenow.mdx
new file mode 100644
index 0000000..f6373b3
--- /dev/null
+++ b/content/docs/adf-databricks/connectors/servicenow.mdx
@@ -0,0 +1,6 @@
+---
+title: ServiceNow
+description: Azure Data Factory lands ServiceNow tables in storage, and Databricks reads the landed files into tables.
+---
+
+../../../shared/connectors/servicenow.mdx
diff --git a/content/docs/adf-databricks/connectors/sftp.mdx b/content/docs/adf-databricks/connectors/sftp.mdx
new file mode 100644
index 0000000..b85d11d
--- /dev/null
+++ b/content/docs/adf-databricks/connectors/sftp.mdx
@@ -0,0 +1,6 @@
+---
+title: SFTP
+description: Azure Data Factory lands SFTP files in storage, and Databricks reads the landed files into tables.
+---
+
+../../../shared/connectors/sftp.mdx
diff --git a/content/docs/adf-databricks/connectors/sharepoint-online-list.mdx b/content/docs/adf-databricks/connectors/sharepoint-online-list.mdx
new file mode 100644
index 0000000..cbc10ac
--- /dev/null
+++ b/content/docs/adf-databricks/connectors/sharepoint-online-list.mdx
@@ -0,0 +1,6 @@
+---
+title: SharePoint Online List
+description: Azure Data Factory lands SharePoint Online List tables in storage, and Databricks reads the landed files into tables.
+---
+
+../../../shared/connectors/sharepoint-online-list.mdx
diff --git a/content/docs/adf-databricks/connectors/shopify.mdx b/content/docs/adf-databricks/connectors/shopify.mdx
new file mode 100644
index 0000000..9e2f992
--- /dev/null
+++ b/content/docs/adf-databricks/connectors/shopify.mdx
@@ -0,0 +1,6 @@
+---
+title: Shopify
+description: Azure Data Factory lands Shopify tables in storage, and Databricks reads the landed files into tables.
+---
+
+../../../shared/connectors/shopify.mdx
diff --git a/content/docs/adf-databricks/connectors/snowflake.mdx b/content/docs/adf-databricks/connectors/snowflake.mdx
new file mode 100644
index 0000000..2a806c0
--- /dev/null
+++ b/content/docs/adf-databricks/connectors/snowflake.mdx
@@ -0,0 +1,6 @@
+---
+title: Snowflake
+description: Azure Data Factory lands Snowflake tables in storage, and Databricks reads the landed files into tables.
+---
+
+../../../shared/connectors/snowflake.mdx
diff --git a/content/docs/adf-databricks/connectors/spark.mdx b/content/docs/adf-databricks/connectors/spark.mdx
new file mode 100644
index 0000000..aa6d7d6
--- /dev/null
+++ b/content/docs/adf-databricks/connectors/spark.mdx
@@ -0,0 +1,6 @@
+---
+title: Spark
+description: Azure Data Factory lands Spark tables in storage, and Databricks reads the landed files into tables.
+---
+
+../../../shared/connectors/spark.mdx
diff --git a/content/docs/adf-databricks/connectors/sqlserver.mdx b/content/docs/adf-databricks/connectors/sqlserver.mdx
new file mode 100644
index 0000000..0fcfdce
--- /dev/null
+++ b/content/docs/adf-databricks/connectors/sqlserver.mdx
@@ -0,0 +1,6 @@
+---
+title: SQL Server
+description: Azure Data Factory lands a full table or a time window of SQL Server in storage, and Databricks reads the landed files into tables.
+---
+
+../../../shared/connectors/sqlserver.mdx
diff --git a/content/docs/adf-databricks/connectors/square.mdx b/content/docs/adf-databricks/connectors/square.mdx
new file mode 100644
index 0000000..b32a8ed
--- /dev/null
+++ b/content/docs/adf-databricks/connectors/square.mdx
@@ -0,0 +1,6 @@
+---
+title: Square
+description: Azure Data Factory lands Square tables in storage, and Databricks reads the landed files into tables.
+---
+
+../../../shared/connectors/square.mdx
diff --git a/content/docs/adf-databricks/connectors/sybase.mdx b/content/docs/adf-databricks/connectors/sybase.mdx
new file mode 100644
index 0000000..f9e5154
--- /dev/null
+++ b/content/docs/adf-databricks/connectors/sybase.mdx
@@ -0,0 +1,6 @@
+---
+title: Sybase
+description: Azure Data Factory lands Sybase tables in storage, and Databricks reads the landed files into tables.
+---
+
+../../../shared/connectors/sybase.mdx
diff --git a/content/docs/adf-databricks/connectors/teradata.mdx b/content/docs/adf-databricks/connectors/teradata.mdx
new file mode 100644
index 0000000..8cd4199
--- /dev/null
+++ b/content/docs/adf-databricks/connectors/teradata.mdx
@@ -0,0 +1,6 @@
+---
+title: Teradata
+description: Azure Data Factory lands Teradata tables in storage, and Databricks reads the landed files into tables.
+---
+
+../../../shared/connectors/teradata.mdx
diff --git a/content/docs/adf-databricks/connectors/vertica.mdx b/content/docs/adf-databricks/connectors/vertica.mdx
new file mode 100644
index 0000000..a17d59a
--- /dev/null
+++ b/content/docs/adf-databricks/connectors/vertica.mdx
@@ -0,0 +1,6 @@
+---
+title: Vertica
+description: Azure Data Factory lands Vertica tables in storage, and Databricks reads the landed files into tables.
+---
+
+../../../shared/connectors/vertica.mdx
diff --git a/content/docs/adf-databricks/connectors/web-table.mdx b/content/docs/adf-databricks/connectors/web-table.mdx
new file mode 100644
index 0000000..f86cc1a
--- /dev/null
+++ b/content/docs/adf-databricks/connectors/web-table.mdx
@@ -0,0 +1,6 @@
+---
+title: Web table
+description: Azure Data Factory lands Web table tables in storage, and Databricks reads the landed files into tables.
+---
+
+../../../shared/connectors/web-table.mdx
diff --git a/content/docs/adf-databricks/connectors/xero.mdx b/content/docs/adf-databricks/connectors/xero.mdx
new file mode 100644
index 0000000..fa59d4a
--- /dev/null
+++ b/content/docs/adf-databricks/connectors/xero.mdx
@@ -0,0 +1,6 @@
+---
+title: Xero
+description: Azure Data Factory lands Xero tables in storage, and Databricks reads the landed files into tables.
+---
+
+../../../shared/connectors/xero.mdx
diff --git a/content/docs/adf-databricks/meta.json b/content/docs/adf-databricks/meta.json
new file mode 100644
index 0000000..17a5b39
--- /dev/null
+++ b/content/docs/adf-databricks/meta.json
@@ -0,0 +1,5 @@
+{
+ "title": "Data Factory into Databricks",
+ "description": "A factory copies each source into landing storage; Databricks pipelines load the tables.",
+ "pages": ["(start)", "(guides)", "connectors"]
+}
diff --git a/content/docs/adf-duckdb/(guides)/deploy.mdx b/content/docs/adf-duckdb/(guides)/deploy.mdx
new file mode 100644
index 0000000..843a695
--- /dev/null
+++ b/content/docs/adf-duckdb/(guides)/deploy.mdx
@@ -0,0 +1,222 @@
+---
+title: Deploy both
+description: Create the DuckDB job once, then deploy each environment's factory and upload its SQL, by hand or from CI; and what to watch when it runs.
+---
+
+Ingestron writes the factory and the SQL and creates nothing in Azure. Create
+each environment's Container Apps job once, then deploy the factory and upload
+the SQL for each change.
+
+## Before you deploy [#prerequisites]
+
+Ingestron creates nothing in Azure. `ingestron request --env dev` lists the
+job, its access and its limits, and puts the role ADF needs to start it under
+**ADF prerequisites**:
+
+
+
+The same request lists what the factory needs; see
+[Before you deploy](/docs/adf/deploy#prerequisites) on Azure Data Factory.
+
+### Create the job
+
+The job runs in a Container Apps environment on the consumption plan. Create
+one if you have none:
+
+```sh
+az containerapp env create --name cae-data --resource-group rg-data --location
+```
+
+Replace `` with the Azure region that holds your storage.
+
+Then deploy the job from the template,
+[`duckdb-job.bicep`](https://github.com/ingestron/docs/blob/main/ci-templates/azure/duckdb-job.bicep):
+
+```sh
+az deployment group create --resource-group rg-data \
+ --template-file duckdb-job.bicep \
+ --parameters name=sales-dev-duckdb \
+ environmentId=$(az containerapp env show -n cae-data -g rg-data --query id -o tsv) \
+ sqlUrl=abfss://deploy@salesdev.dfs.core.windows.net/sales/dev/run.sql \
+ storageAccountName=salesdev \
+ factoryPrincipalId=$(az datafactory show -n adf-sales-dev -g rg-data --query identity.principalId -o tsv)
+```
+
+Its parameters:
+
+
+
+The template creates the job with:
+
+- **Identity:** its own managed identity, with Storage Blob Data Contributor on
+ the storage account, to read the landed files and `run.sql` and to publish
+ tables.
+- **Resources and limits:** 0.5 vCPU and 1 GiB per run, a replica timeout of
+ 1800 seconds, and one retry. Raise the CPU and memory for larger tables, up
+ to the consumption plan's limits.
+- **Starting and stopping:** the Container Apps Jobs Operator role on the job
+ for your data factory, so ADF can start runs, read them and stop them. The
+ role also lets the factory change a run's container, so the job holds no
+ secrets.
+- **Image:** the public image, pulled anonymously. `:1` moves to each new 1.x
+ release of the runner; to choose when it changes, pass `image=` with a full
+ version tag or a digest (`ghcr.io/ingestron/duckdb-runner@sha256:…`). If your
+ organisation only allows its own registry, copy the image there and pass
+ `image=`; the template shows where the registry's settings go. A private
+ registry adds its own cost.
+
+The template outputs the job's resource ID, `jobId`, for `after_copy`.
+
+## Deploy [#deploy]
+
+
+
+Run `ingestron apply`, deploy the factory as in
+[Build the template and deploy it](/docs/adf/deploy#build-and-deploy), then copy the environment's
+DuckDB folder to the folder `sqlUrl` names, here for `dev`:
+
+```sh
+az storage fs directory upload --account-name salesdev --file-system deploy \
+ --source "build/duckdb/dev/*" --destination-path sales/dev --recursive --auth-mode login
+```
+
+The cloud environment's `run.sql` must read and write `abfss://` paths: the job
+refuses one that names local folders. A local environment, such as `local`
+here, is never deployed.
+
+
+
+### From CI [#deploy-from-ci]
+
+Each environment's stage deploys the factory, then uploads that environment's
+folder. From the GitHub Actions template:
+
+
+
+The job itself, from `duckdb-job.bicep`, is created once per environment and
+is not part of each deployment.
+[Deploy the factory](/docs/adf/deploy#deploy-from-ci) gives the factory's
+steps.
+
+## Run and operate [#operate]
+
+
+
+The schedule's trigger starts the parent pipeline each window; outside
+production it is stopped, as the
+[platform defaults](/docs/reference/packs/adf#platform-configuration) say. Each
+job run shows in the job's execution history:
+
+
+
+- **A run that never finishes** is stopped at
+ [`after_copy.timeout`](/docs/reference/packs/adf#adf-copy-after-copy-timeout),
+ and the pipeline fails, saying whether the run stopped. The job's
+ own replica timeout is half of that, so a retried run also ends in time.
+- **A failed run** is retried once by the job. Retries are safe because each
+ run rebuilds everything. ADF never starts the job twice for one load: a
+ failed copy is retried in place, and the schedule's trigger does not retry.
+- **Cost:** consumption-plan jobs bill per second only while a run goes. The
+ first 180,000 vCPU-seconds and 360,000 GiB-seconds per subscription each
+ month are free
+ ([Container Apps pricing](https://azure.microsoft.com/en-us/pricing/details/container-apps/),
+ checked 4 November 2026), so a small daily job may cost nothing. ADF bills
+ each activity run: the wait reads the run's status every 30 seconds, about
+ three activity runs a poll.
+- **Every run reads every landed file.** With `layout: snapshot` that is one
+ file per table. With increments, each run reads them all; keep them bounded,
+ knowing that deleting old files removes their rows from `append` tables.
+- **Two schedules** may start the job at once. Each run rebuilds everything,
+ and the later run's files win.
+
+When the parent pipeline fails, see
+[The job after the copies fails or times out](#the-job-after-the-copies-fails-or-times-out).
+
+### When something fails [#troubleshooting]
+
+#### The job after the copies fails or times out
+
+**Symptom:** the parent pipeline `pl__every__` fails
+in Data Factory with one of:
+
+```text
+The job after the copies ended Failed.
+The job after the copies did not finish within 01:00:00, so it was stopped.
+The job after the copies did not finish within 01:00:00 and could not be stopped:
+```
+
+The status can be another word than `Failed`, and the time is the
+`after_copy` timeout.
+
+**Cause:** the Container Apps job ended without succeeding, or ran past the
+timeout (one hour by default). The pipeline reads the run's status every 30
+seconds. The last message means ADF could not stop the run: the factory's
+identity may lack the **Container Apps Jobs Operator** role on the job.
+
+**Fix:** open the job's run in Azure for the error. The job retries a failed run
+when its retry limit is 1, as `ingestron request` asks, and running again is
+safe because each run rebuilds everything.
+Raise `after_copy.timeout` for a larger load. `ingestron request --env dev`
+lists the job and the access it needs. See
+[Run and operate](#operate).
+
+#### Append would miss later loads
+
+**Symptom:**
+
+```text
+flows/erp.yaml:6:13 error METHOD_CHECK
+ "duckdb-table" for table "invoices": append reads each file once, but "adf-copy" replaces its file each run, so later loads would be missed; use replace or upsert.
+```
+
+**Cause:** `append` reads each file name once. A copy that replaces its file
+under the same name every load would never be read again. For the same reason,
+a file you re-export under its old name is skipped.
+
+**Fix:** keep the table with `history: none` (the method's `replace`) or
+`history: latest` (`upsert`) instead of `mode: append`, or give each load its
+own file, such as with `increments`. To reload everything, delete `ingestron.duckdb` and run
+again. See [Keep a table current](/docs/duckdb/keep-a-table-current#on-duckdb).
+
+#### Paths mix local and Azure
+
+**Symptom:** the environment's `README.md` begins:
+
+```text
+This environment reads or writes both local paths and paths in Azure storage,
+and local paths cannot run in Azure: a job there cannot see them. Give it only
+paths in Azure storage, or only local paths. `run.sql` has no Azure sign-in of
+its own.
+```
+
+and `ingestron request` adds `give this environment only paths in Azure storage`.
+
+**Cause:** one path in the environment is local, such as `./published`, and
+another is in Azure. A job in Azure cannot see a local path, and `run.sql`
+has no Azure sign-in of its own, so a laptop cannot read the Azure ones.
+
+**Fix:** give the environment only Azure paths, or only local ones. To try a
+cloud project on a laptop, use an environment with `runs-on: duckdb`. See
+[Run a cloud project locally](/docs/adf-duckdb/load-with-adf#run-a-cloud-project-locally).
+
+## Next
+
+- [Load with Azure Data Factory](/docs/adf-duckdb/load-with-adf): what each
+ part writes, and the settings that shape it.
+- [Run the factory](/docs/adf/deploy#operate).
diff --git a/content/docs/platforms/duckdb-in-azure.mdx b/content/docs/adf-duckdb/(guides)/load-with-adf.mdx
similarity index 50%
rename from content/docs/platforms/duckdb-in-azure.mdx
rename to content/docs/adf-duckdb/(guides)/load-with-adf.mdx
index d8f88a9..b8a267e 100644
--- a/content/docs/platforms/duckdb-in-azure.mdx
+++ b/content/docs/adf-duckdb/(guides)/load-with-adf.mdx
@@ -1,6 +1,6 @@
---
-title: Run DuckDB in Azure
-description: Rebuild a project's DuckDB tables in Azure after each ADF load, with a Container Apps job and a public image.
+title: Load with Azure Data Factory
+description: Let Azure Data Factory land the files, and rebuild the DuckDB tables in Azure after each load with a Container Apps job and a public image.
---
When ADF lands the files and DuckDB builds the tables, an Azure Container Apps
@@ -94,16 +94,17 @@ cd sales
-Run **Ingestron: New project**, choose **SQL Server** and the ADF route into
-DuckDB tables, then the folder.
+New project starts a project for one platform. To have Azure Data Factory
+load the files DuckDB reads, create the project in a terminal with the command
+on the CLI tab, then open its folder in VS Code.
`init` writes `REPLACE_ME`, with a comment, for each value it cannot know,
such as the server, the tables and the storage account, and `README.md` lists
-them; see [Write the values it cannot know](/docs/how-to/start-from-a-template#write-the-values).
-For another source, choose it from the [connector catalogue](/docs/connectors);
+them; see [Write the values it cannot know](/docs/duckdb/create-a-project#write-the-values).
+For another source, choose it from the [connector catalogue](/docs/~/connectors);
its page lists the routes it supports. The files on this page come from a
finished project, which also runs on a laptop:
@@ -140,159 +141,80 @@ When you turn `after_copy` on in an environment that already runs:
- delete the connection pipelines' old triggers from the factory, because an
incremental publish leaves them running, which would copy everything twice.
-The tables' modes are as on [DuckDB](/docs/platforms/duckdb#settings), and the
+The tables' modes are as on [DuckDB](/docs/duckdb#settings), and the
copies' schedule and runtime as on
-[Azure Data Factory](/docs/platforms/adf#runtime-and-key-vault).
+[Azure Data Factory](/docs/adf#runtime-and-key-vault).
`after_copy.job` names a resource in one environment, so give each environment
its job's ID in `project.yaml`: where the IDs differ, the factory takes the
job as a template parameter.
-## Before you deploy [#prerequisites]
+### Run a cloud project locally [#run-a-cloud-project-locally]
-Ingestron creates nothing in Azure. `ingestron request --env dev` lists the
-job, its access and its limits, and puts the role ADF needs to start it under
-**ADF prerequisites**:
+
-
+When ADF lands the files and DuckDB builds the tables, an environment that names
+`runs-on: duckdb` runs the same tables on a laptop:
-See [Ask the platform team](/docs/how-to/ask-the-platform-team) for the whole
-request.
+
-### Create the job
+In `local`, ADF's copy becomes `duckdb-landed`, which reads each table's files
+from `/