Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
206 changes: 179 additions & 27 deletions .github/workflows/test.main.kts
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,10 @@ import io.github.typesafegithub.workflows.dsl.expressions.expr
import io.github.typesafegithub.workflows.dsl.workflow
import io.github.typesafegithub.workflows.yaml.CheckoutActionVersionSource
import io.github.typesafegithub.workflows.yaml.DEFAULT_CONSISTENCY_CHECK_JOB_CONFIG
import it.krzeminski.snakeyaml.engine.kmp.api.Dump
import it.krzeminski.snakeyaml.engine.kmp.api.DumpSettings
import it.krzeminski.snakeyaml.engine.kmp.api.Load
import it.krzeminski.snakeyaml.engine.kmp.common.FlowStyle
import kotlinx.serialization.json.JsonArray
import kotlinx.serialization.json.JsonElement
import kotlinx.serialization.json.JsonNull
Expand All @@ -52,6 +55,28 @@ import kotlin.io.path.extension
import kotlin.io.path.invariantSeparatorsPathString
import kotlin.io.path.name
import kotlin.io.path.walk
import kotlin.system.exitProcess

private val mismatchBaselinePath = "typing-mismatches.yml"

private val mismatchBaselineHeader = listOf(
"# Generated with: DUMP_TYPING_MISMATCHES=true .github/workflows/test.main.kts",
"# Lists all differences between the typings in this catalog and the types inferred by the heuristics.",
"# CI fails when this file is out of sync with the actual mismatches.",
).joinToString("\n")

private val mismatchComparator = compareBy<TypingMismatch> { it.action.owner }
.thenBy { it.action.name }
.thenBy { versionOrder(it.action.version) }
.thenBy { it.action.version }
.thenBy { it.action.path ?: "" }
.thenBy { sectionOrder(it.inference.section) }
.thenBy { it.inference.name }

if (System.getenv("DUMP_TYPING_MISMATCHES") == "true") {
dumpTypingMismatchBaseline()
exitProcess(0)
}

workflow(
name = "Test",
Expand Down Expand Up @@ -170,6 +195,30 @@ private data class TypingInference(
val isMismatch: Boolean get() = actual != inferred
}

private data class TypingMismatch(
val action: ActionCoords,
val inference: TypingInference,
) {
val key: MismatchKey = MismatchKey(action.key, inference.section, inference.name)
val values: MismatchValues = MismatchValues(inference.actual, inference.inferred)
}

private data class MismatchKey(
val action: String,
val section: String,
val name: String,
) {
override fun toString(): String = "$action:$section:$name"
}

private data class MismatchValues(
val actual: Map<String, Any?>,
val inferred: Map<String, Any?>,
)

private val ActionCoords.key: String
get() = listOfNotNull("$owner/$name@$version", path).joinToString("/")

private fun resolveBaseRef(rawBaseRef: String?, ref: String?, repository: String?): String? {
if (rawBaseRef == null || !rawBaseRef.startsWith("graphite-base/")) return rawBaseRef

Expand Down Expand Up @@ -198,20 +247,21 @@ private fun validateTypings(sha: String, baseRef: String?) {
)

println()
val actionVersions = listActionVersionsToValidate(sha = sha, baseRef = baseRef)

var shouldFail = false
val actionVersions = listActionVersionsToValidate(sha = sha, baseRef = baseRef).toList()

val typingDifferences = mutableListOf<TypingDifference>()
val validationErrors = mutableListOf<String>()
val validatedActionKeys = mutableSetOf<String>()
val mismatches = mutableListOf<TypingMismatch>()

for (action in actionVersions) {
println()
println("➡\uFE0F For https://github.com/${action.owner}/${action.name}/tree/${action.version}/${action.path ?: ""}")
val actionDescription =
"https://github.com/${action.owner}/${action.name}/tree/${action.version}/${action.path ?: ""}"
println("➡\uFE0F For $actionDescription")

if (action.pathToTypings != action.pathToTypings.lowercase()) {
println("\uD83D\uDD34 Action's owner and name should be lowercase, " +
"to enable the bindings server to load them in a case-insensitive manner!")
shouldFail = true
validationErrors += "$actionDescription: action's owner and name should be lowercase, " +
"to enable the bindings server to load them in a case-insensitive manner!"
continue
}

Expand All @@ -224,9 +274,8 @@ private fun validateTypings(sha: String, baseRef: String?) {

val schemaComplianceErrors = typingsSchema.checkForSchemaComplianceErrors(typings)
if (schemaComplianceErrors != null) {
println("\uD83D\uDD34 Typings aren't compliant with the schema!")
println(schemaComplianceErrors)
shouldFail = true
validationErrors += "$actionDescription: typings aren't compliant with the schema: " +
schemaComplianceErrors.replace("\n", " ")
continue
}

Expand All @@ -235,15 +284,17 @@ private fun validateTypings(sha: String, baseRef: String?) {
val manifest = fetchManifest(action)

if (manifest == null) {
println("\uD83D\uDD34 No manifest found!")
shouldFail = true
validationErrors += "$actionDescription: no manifest found"
continue
}

val manifestInputs = manifest.names("inputs")
val manifestOutputs = manifest.names("outputs")

val inferences = compareTypings(typings, manifest)
validatedActionKeys += action.key
mismatches += inferences.filter { it.isMismatch }.map { TypingMismatch(action, it) }

inferences.forEach { inference ->
val warning = if (inference.isMismatch) "\u26A0\uFE0F " else ""
val reason = inference.reason?.let { " ($it)" } ?: ""
Expand All @@ -255,26 +306,33 @@ private fun validateTypings(sha: String, baseRef: String?) {
}

if (typingsInputs != manifestInputs || typingsOutputs != manifestOutputs) {
println("\uD83D\uDD34 Something is wrong with the typings!")

val difference = TypingDifference(
action = action,
extraInputsInManifest = manifestInputs - typingsInputs,
extraInputsInTypings = typingsInputs - manifestInputs,
extraOutputsInManifest = manifestOutputs - typingsOutputs,
extraOutputsInTypings = typingsOutputs - manifestOutputs
)
println(difference)

typingDifferences += difference
shouldFail = true
validationErrors += "$actionDescription: something is wrong with the typings: " +
TypingDifference(
action = action,
extraInputsInManifest = manifestInputs - typingsInputs,
extraInputsInTypings = typingsInputs - manifestInputs,
extraOutputsInManifest = manifestOutputs - typingsOutputs,
extraOutputsInTypings = typingsOutputs - manifestOutputs,
)
continue
}

println("\uD83D\uDFE2 OK!")
}

require(shouldFail == false) {
val baselineIssues = validateMismatchBaseline(validatedActionKeys, mismatches)
validationErrors += baselineIssues

if (validationErrors.isNotEmpty()) {
println()
println("\uD83D\uDD34 Typing validation found ${validationErrors.size} issue(s):")
validationErrors.sorted().forEach { println(" - $it") }
if (baselineIssues.isNotEmpty()) {
println(" Regenerate with: DUMP_TYPING_MISMATCHES=true .github/workflows/test.main.kts")
}
}

require(validationErrors.isEmpty()) {
"This is the end of processing, and something doesn't match - see the logs!"
}
}
Expand Down Expand Up @@ -434,6 +492,10 @@ private fun formatTypingValue(value: Any?): String = when (value) {
private fun Map<*, *>.names(section: String): Set<String> =
(this[section] as? Map<*, *>)?.keys?.map { it.toString() }?.toSet() ?: emptySet()

/**
* Single source of truth for comparing stored typings with heuristic inferences. Both validation and
* the baseline dump go through here, so the YAML file can't drift from what CI checks.
*/
private fun compareTypings(typings: Map<*, *>, manifest: Map<*, *>): List<TypingInference> =
// Input names feed boolean heuristics (e.g. `skip-cache`). Outputs have names too, but
// update-typings.main.kts currently omits passing them - that's a bug there. Mirror it for now
Expand Down Expand Up @@ -463,6 +525,96 @@ private fun normalizeTyping(value: Any?): Any? = when (value) {
else -> value
}

private fun validateMismatchBaseline(
validatedActionKeys: Set<String>,
mismatches: List<TypingMismatch>,
): List<String> {
val expected = loadMismatchBaseline().filterKeys { it.action in validatedActionKeys }
val actual = mismatches.associate { it.key to it.values }

val unexpected = actual.keys - expected.keys
val stale = expected.keys - actual.keys
val changed = actual.keys.intersect(expected.keys).filter { actual[it] != expected[it] }

val issues = mutableListOf<String>()
unexpected.sortedBy { it.toString() }.forEach {
issues += "$mismatchBaselinePath is missing a mismatch for $it: ${formatMismatchValues(actual[it]!!)}"
}
stale.sortedBy { it.toString() }.forEach {
issues += "$mismatchBaselinePath has a stale entry for $it: the inferred typing now matches the actual typing"
}
changed.sortedBy { it.toString() }.forEach { key ->
issues += "$mismatchBaselinePath has outdated typings for $key: baseline " +
"${formatMismatchValues(expected[key]!!)}, actual ${formatMismatchValues(actual[key]!!)}"
}
return issues
}

private fun formatMismatchValues(values: MismatchValues): String =
"actual: {${formatTyping(values.actual)}}, inferred: {${formatTyping(values.inferred)}}"

private fun loadMismatchBaseline(): Map<MismatchKey, MismatchValues> {
val file = File(mismatchBaselinePath)
if (!file.exists()) return emptyMap()
val root = Load().loadOne(file.readText()) as? Map<*, *> ?: return emptyMap()
val actions = root["actions"] as? Map<*, *> ?: return emptyMap()
return buildMap {
actions.forEach { (actionKey, actionNode) ->
(actionNode as? Map<*, *>)?.forEach { (section, sectionNode) ->
(sectionNode as? Map<*, *>)?.forEach { (name, values) ->
val valueMap = values as? Map<*, *> ?: return@forEach
put(
MismatchKey(actionKey.toString(), section.toString(), name.toString()),
MismatchValues(
actual = (valueMap["actual"] as? Map<*, *>)?.asTypingMap() ?: emptyMap(),
inferred = (valueMap["inferred"] as? Map<*, *>)?.asTypingMap() ?: emptyMap(),
),
)
}
}
}
}
}

private fun dumpTypingMismatchBaseline() {
val mismatches = listAllActionManifestFilesInRepo()
.map { it.toActionCoords() }
.toList()
.flatMap { action ->
val manifest = fetchManifest(action)
if (manifest == null) {
println("\u26A0\uFE0F No manifest found for ${action.key}, skipping")
emptyList()
} else {
compareTypings(loadTypings(action.pathToTypings), manifest)
.filter { it.isMismatch }
.map { TypingMismatch(action, it) }
}
}
.sortedWith(mismatchComparator)

File(mismatchBaselinePath).writeText(renderMismatchBaseline(mismatches))
println("Wrote ${mismatches.size} mismatches to $mismatchBaselinePath")
}

private fun renderMismatchBaseline(mismatches: List<TypingMismatch>): String {
val actions = linkedMapOf<String, Any>()
mismatches.forEach { mismatch ->
val actionNode = actions.getOrPut(mismatch.action.key) { linkedMapOf<String, Any>() } as MutableMap<String, Any>
val sectionNode = actionNode.getOrPut(mismatch.inference.section) { linkedMapOf<String, Any>() } as MutableMap<String, Any>
sectionNode[mismatch.inference.name] = linkedMapOf(
"actual" to mismatch.inference.actual,
"inferred" to mismatch.inference.inferred,
)
}
val settings = DumpSettings(defaultFlowStyle = FlowStyle.BLOCK, indent = 2, width = 120)
return mismatchBaselineHeader + "\n" + Dump(settings).dumpToString(mapOf("actions" to actions))
}

private fun versionOrder(version: String): Int = version.removePrefix("v").toIntOrNull() ?: Int.MAX_VALUE

private fun sectionOrder(section: String): Int = if (section == "inputs") 0 else 1

private fun fetchManifest(action: ActionCoords): Map<String, Any>? {
val list = listOf(action.actionYmlUrl, action.actionYamlUrl)

Expand Down
Loading