Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions docs/src/components/docs-search/search-logic.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -314,6 +314,12 @@ describe("snippet", () => {
expect(result.endsWith("\u2026")).toBe(true);
});

it("centers on the match after text whose lowercase is longer, such as a dotted capital I", () => {
// "İ".toLowerCase() is two characters: an offset found in a lowercased copy lands past the match.
const text = "İ".repeat(100) + " match " + "b".repeat(40);
expect(snippet(text, "match", tokens("match"))).toContain("match");
});

it("keeps output to roughly 120 characters of text", () => {
const text = "word ".repeat(100);
const result = snippet(text, "word", tokens("word"));
Expand Down
15 changes: 15 additions & 0 deletions docs/src/components/docs-search/search-logic.ts
Original file line number Diff line number Diff line change
Expand Up @@ -171,6 +171,20 @@ export function score(e: SearchEntry, q: string, toks: string[]): number {
return 0;
}

/**
* Where offset `idx` of `lower` (`text.toLowerCase()`) falls in `text`. Lowercasing can lengthen a string ("İ"
* becomes two characters), so past such a character the two offsets differ.
*/
function indexInOriginal(text: string, lower: string, idx: number): number {
if (idx <= 0 || lower.length === text.length) return idx;
var lowered = 0;
for (var i = 0; i < text.length; i++) {
if (lowered >= idx) return i;
lowered += text[i].toLowerCase().length;
}
return text.length;
}

/** Returns ~120 chars centered on the first query/token match in text. */
export function snippet(
text: string | null | undefined,
Expand All @@ -187,6 +201,7 @@ export function snippet(
if (idx >= 0) break;
}
}
idx = indexInOriginal(text, lo, idx);
var start = Math.max(0, idx > 40 ? idx - 40 : 0);
return (
(start > 0 ? "\u2026" : "") +
Expand Down
25 changes: 23 additions & 2 deletions lsp/src/preserved-targets.ts
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,28 @@ const directiveBits = (value: string | undefined): number => {
};

/** The offset just past the `>` closing a start tag that begins at `from`, honoring quotes; -1 when unclosed. */
/**
* The index of `</name` at or after `from`, in any ASCII case, or -1. `name` is lowercase letters. Tag names are
* compared in place, as an HTML parser compares them: a lowercased copy of the markup can be longer than it ("İ"
* becomes two characters), so an index found there points past the real end tag.
*/
const findEndTag = (markup: string, name: string, from: number): number => {
for (
let at = markup.indexOf("</", from);
at !== -1;
at = markup.indexOf("</", at + 2)
) {
let k = 0;
while (
k < name.length &&
(markup.charCodeAt(at + 2 + k) | 0x20) === name.charCodeAt(k)
)
k++;
if (k === name.length) return at;
}
return -1;
};

const startTagEnd = (markup: string, from: number): number => {
let quote = "";
for (let i = from; i < markup.length; i++) {
Expand All @@ -118,7 +140,6 @@ export const collectPreservedTargets = (
preserve: readonly string[],
): PreservedTargets => {
const matchers = preserveMatchers(preserve);
const lower = markup.toLowerCase();
const stack: { name: string; bits: number }[] = [];
const preserved = {
classes: new Set<string>(),
Expand Down Expand Up @@ -196,7 +217,7 @@ export const collectPreservedTargets = (
classTarget.tags.add(name);

if (RAW_TEXT_ELEMENTS.has(name)) {
const rawEnd = lower.indexOf(`</${name}`, end);
const rawEnd = findEndTag(markup, name, end);
i = rawEnd === -1 ? markup.length : rawEnd;
continue;
}
Expand Down
18 changes: 18 additions & 0 deletions pkg/src/lib/generated-id-classes.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -157,6 +157,24 @@ describe("generated #id class collisions", () => {
expect(error.message.match(/^error: /gm)).toHaveLength(2);
});

it("finds a <style> rule after text whose lowercase is longer, such as a dotted capital I", async () => {
// "İ".toLowerCase() is two characters: an offset found in a lowercased copy of the markup lands past the rule.
write(
"src/components/site-nav_x.html",
'<nav><button id="btn">Menu</button></nav>\n',
);
write("src/components/site-nav_x.css", "#btn { color: red; }\n");
write(
"src/components/site-nav.html",
`<nav>\n <a id="x_btn">${"İ".repeat(40)}</a>\n</nav>\n<style>\n #x_btn { color: blue; }\n</style>\n`,
);

const error = await collisionOf();
expect(error.message).toContain(
"src/components/site-nav.html:5:3 #x_btn in <site-nav>",
);
});

it("reads CRLF line endings as one line each", async () => {
write("src/components/one_two.html", '<p id="three">a</p>');
write(
Expand Down
34 changes: 30 additions & 4 deletions pkg/src/lib/generated-id-classes.ts
Original file line number Diff line number Diff line change
Expand Up @@ -133,16 +133,42 @@ const lineAndColumn = (
return { line, column: offset - lineStart + 1 };
};

/**
* The index of `tag` (`<` and lowercase ASCII, such as `</style`) at or after `from` in `html`, in any ASCII case,
* or -1. Compared in place: a lowercased copy of `html` can be longer than it ("İ" becomes two characters), so an
* index found there points past the text it names.
*/
const indexOfTag = (html: string, tag: string, from: number): number => {
for (
let at = html.indexOf("<", from);
at !== -1;
at = html.indexOf("<", at + 1)
) {
let k = 1;
for (; k < tag.length; k++) {
const code = html.charCodeAt(at + k);
const wanted = tag.charCodeAt(k);
// A lowercase letter also matches its uppercase; anything else only itself.
if (
code !== wanted &&
!(wanted >= 97 && wanted <= 122 && code === wanted - 32)
)
break;
}
if (k === tag.length) return at;
}
return -1;
};

/** The offset of the first `#id` rule naming `idName` inside a `<style>` element of `html`, or -1. */
const styleElementOffset = (html: string, idName: string): number => {
const lower = html.toLowerCase();
let from = 0;
for (;;) {
const open = lower.indexOf("<style", from);
const open = indexOfTag(html, "<style", from);
if (open === -1) return -1;
const contentStart = lower.indexOf(">", open);
const contentStart = html.indexOf(">", open);
if (contentStart === -1) return -1;
const close = lower.indexOf("</style", contentStart);
const close = indexOfTag(html, "</style", contentStart);
const contentEnd = close === -1 ? html.length : close;
const offset = cssIdSelectorOffset(
html.slice(contentStart + 1, contentEnd),
Expand Down
21 changes: 21 additions & 0 deletions pkg/src/lib/preserved-targets.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -271,6 +271,27 @@ document.querySelector(".inline");`;
});
});

describe("collectPreservedTargets: raw-text elements", () => {
it("finds the end of a raw-text element in any ASCII case", () => {
const targets = collectPreservedTargets(
'<TITLE>x</Title ><p class="kept">a</p><STYLE>p{}</STYLE><p class="also">b</p>',
["p"],
);
expect([...targets.classes]).toEqual(["kept", "also"]);
});

it("reads past a raw-text element whose lowercase is longer, such as one holding a dotted capital I", () => {
// "İ".toLowerCase() is two characters: an end found in a lowercased copy of the markup lands past the next tag.
for (const element of ["title", "style", "textarea", "script"]) {
const targets = collectPreservedTargets(
`<${element}>${"İ".repeat(12)}</${element}><p class="kept">a</p>`,
["p"],
);
expect([...targets.classes], element).toEqual(["kept"]);
}
});
});

describe("preserved-targets copies", () => {
it("keeps the language server's copy byte-identical to the compiler's", () => {
const pkgCopy = readFileSync(
Expand Down
25 changes: 23 additions & 2 deletions pkg/src/lib/preserved-targets.ts
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,28 @@ const directiveBits = (value: string | undefined): number => {
};

/** The offset just past the `>` closing a start tag that begins at `from`, honoring quotes; -1 when unclosed. */
/**
* The index of `</name` at or after `from`, in any ASCII case, or -1. `name` is lowercase letters. Tag names are
* compared in place, as an HTML parser compares them: a lowercased copy of the markup can be longer than it ("İ"
* becomes two characters), so an index found there points past the real end tag.
*/
const findEndTag = (markup: string, name: string, from: number): number => {
for (
let at = markup.indexOf("</", from);
at !== -1;
at = markup.indexOf("</", at + 2)
) {
let k = 0;
while (
k < name.length &&
(markup.charCodeAt(at + 2 + k) | 0x20) === name.charCodeAt(k)
)
k++;
if (k === name.length) return at;
}
return -1;
};

const startTagEnd = (markup: string, from: number): number => {
let quote = "";
for (let i = from; i < markup.length; i++) {
Expand All @@ -118,7 +140,6 @@ export const collectPreservedTargets = (
preserve: readonly string[],
): PreservedTargets => {
const matchers = preserveMatchers(preserve);
const lower = markup.toLowerCase();
const stack: { name: string; bits: number }[] = [];
const preserved = {
classes: new Set<string>(),
Expand Down Expand Up @@ -196,7 +217,7 @@ export const collectPreservedTargets = (
classTarget.tags.add(name);

if (RAW_TEXT_ELEMENTS.has(name)) {
const rawEnd = lower.indexOf(`</${name}`, end);
const rawEnd = findEndTag(markup, name, end);
i = rawEnd === -1 ? markup.length : rawEnd;
continue;
}
Expand Down
20 changes: 20 additions & 0 deletions skills-tests/tests/skill-scripts.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -548,6 +548,15 @@ describe("audit-seo", () => {
expect(facts.imagesWithoutAlt).toBe(0);
});

it("reads past a script or style whose lowercase is longer, such as one holding a dotted capital I", () => {
// "İ".toLowerCase() is two characters: an end found in a lowercased copy of the page lands past the next tag.
const facts = inspectPage(
`<body><script>${"İ".repeat(12)}</script><h1>real</h1><style>${"İ".repeat(12)}</style><img src=x></body>`,
);
expect(facts.h1Count).toBe(1);
expect(facts.imagesWithoutAlt).toBe(1);
});

it("does not form new HTML from either side of a script or style", () => {
const facts = inspectPage(
'<body><h<script>inert</script>1>fake</h1><SCRIPT>"<h1>ignored</h1>"</script\t\n data-test><h1>real</h1></body>',
Expand Down Expand Up @@ -822,6 +831,17 @@ describe("check-links", () => {
).toEqual(["/a.png", "/a.png", "/a@2x.png"]);
});

it("reads past a masked element whose lowercase is longer, such as one holding a dotted capital I", () => {
// "İ".toLowerCase() is two characters: an end found in a lowercased copy of the page lands past the next tag.
const links = extractLinks(
`<pre>${"İ".repeat(12)}</pre><a href="/after-pre">a</a><script>${"İ".repeat(12)}</script><a href="/after-script">b</a>`,
);
expect(links.map((link: { url: string }) => link.url)).toEqual([
"/after-pre",
"/after-script",
]);
});

it("decodes entities once and ignores links in masked elements", () => {
const links = extractLinks(
'<!-- <a href="/comment"> --><SCRIPT>"<a href=\'/script\'>"</SCRIPT><pre><a href="/pre">x</a></pre><a href="/a&amp;lt;b&amp;gt;">real</a>',
Expand Down
9 changes: 5 additions & 4 deletions skills/bascik-seo/scripts/audit-seo.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,6 @@ export function inspectPage(html) {
metas.find((m) => (m[key] ?? "").toLowerCase() === value)?.content;
// Preserve boundaries between visible regions, rather than joining text on
// either side of a removed element into a new tag on another pass.
const lower = html.toLowerCase();
const visible = [];
const opening = /<(script|style)\b[^>]*>/gi;
let from = 0;
Expand All @@ -97,9 +96,11 @@ export function inspectPage(html) {
if (match.index < from) continue;
visible.push(html.slice(from, match.index));
const name = match[1].toLowerCase();
let close = lower.indexOf(`</${name}`, opening.lastIndex);
while (close >= 0 && !/[\s/>]/.test(lower[close + name.length + 2] ?? ""))
close = lower.indexOf(`</${name}`, close + name.length + 2);
// Found in the page itself, in ASCII case: a lowercased copy can be longer than
// the page ("İ" becomes two characters), so its offsets point past the tag.
const endTag = new RegExp(`</${name}(?=[\\s/>])`, "gi");
endTag.lastIndex = opening.lastIndex;
const close = endTag.exec(html)?.index ?? -1;
const end = close < 0 ? -1 : html.indexOf(">", close + name.length + 2);
from = end < 0 ? html.length : end + 1;
opening.lastIndex = from;
Expand Down
9 changes: 5 additions & 4 deletions skills/bascik/scripts/check-links.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,6 @@ export function extractLinks(html) {
};
// Scan disjoint visible regions. Removing markup and joining the leftovers can
// turn <scr<script>ipt> into a new <script> on the next pass.
const lower = html.toLowerCase();
let from = 0;
const ignored = /<!--|<(script|template|textarea|code|pre)\b[^>]*>/gi;
let match;
Expand All @@ -76,9 +75,11 @@ export function extractLinks(html) {
const name = match[1].toLowerCase();
// External scripts have a real src link; their contents are still inert.
if (name === "script" && /\bsrc\s*=/i.test(match[0])) scan(match[0]);
let close = lower.indexOf(`</${name}`, ignored.lastIndex);
while (close >= 0 && !/[\s/>]/.test(lower[close + name.length + 2] ?? ""))
close = lower.indexOf(`</${name}`, close + name.length + 2);
// Found in the page itself, in ASCII case: a lowercased copy can be longer than
// the page ("İ" becomes two characters), so its offsets point past the tag.
const endTag = new RegExp(`</${name}(?=[\\s/>])`, "gi");
endTag.lastIndex = ignored.lastIndex;
const close = endTag.exec(html)?.index ?? -1;
const end = close < 0 ? -1 : html.indexOf(">", close + name.length + 2);
from = end < 0 ? html.length : end + 1;
}
Expand Down
Loading