Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 45 additions & 0 deletions .github/scripts/verify_shared_crate_packages.sh
Original file line number Diff line number Diff line change
Expand Up @@ -14,3 +14,48 @@ cargo +stable new --quiet --lib "${consumer_root}/consumer"
cargo +stable add --quiet --manifest-path "${consumer_root}/consumer/Cargo.toml" \
--path "${package_root}/package/fensu-policy-${version}"
cargo +1.85.0 check --locked --manifest-path "${consumer_root}/consumer/Cargo.toml"

# The Python analysis crates are packaged together, as fensu-native depends on fensu-facts. A fresh
# project then runs the core rules without default features, and must not pull in the Python
# bindings or the cache's SQLite and compression libraries.
analysis_version="$(cargo +1.95.0 metadata --locked --no-deps --format-version 1 \
| jq -r '.packages[] | select(.name == "fensu-native") | .version')"
facts_package="${package_root}/package/fensu-facts-${analysis_version}"
native_package="${package_root}/package/fensu-native-${analysis_version}"
analysis_consumer="${consumer_root}/analysis-consumer"

CARGO_TARGET_DIR="${package_root}" cargo +1.95.0 package --locked -p fensu-facts -p fensu-native

cargo +stable new --quiet --bin "${analysis_consumer}"
cat >> "${analysis_consumer}/Cargo.toml" <<TOML
fensu-facts = { path = "${facts_package}", default-features = false }
fensu-native = { path = "${native_package}", default-features = false }

[patch.crates-io]
fensu-facts = { path = "${facts_package}" }
TOML
cat > "${analysis_consumer}/src/main.rs" <<'RUST'
use fensu_facts::ProgramHandle;
use fensu_native::rules::constants::PARAMETER_ANNOTATION_CODE;
use fensu_native::{evaluate_core_rules, NativeProjectPlane, NativeRuleContext};

fn main() {
let program = ProgramHandle::parse_source("def total(items):\n return items\n", 3, 12)
.expect("the module parses");
let faults = evaluate_core_rules(
&program,
&[PARAMETER_ANNOTATION_CODE.to_owned()],
&NativeRuleContext::default(),
&NativeProjectPlane::new(Vec::new(), Vec::new()),
)
.expect("the core rules run");
assert_eq!(faults.len(), 1, "one unannotated parameter: {faults:?}");
assert_eq!(faults[0].code, PARAMETER_ANNOTATION_CODE);
}
RUST
cargo +1.95.0 run --quiet --manifest-path "${analysis_consumer}/Cargo.toml"
analysis_tree="$(cargo +1.95.0 tree --manifest-path "${analysis_consumer}/Cargo.toml" -e normal)"
if grep -E '\b(pyo3|rusqlite|flate2) v' <<< "${analysis_tree}"; then
echo "the analysis crates pulled in Python, SQLite or compression dependencies" >&2
exit 1
fi
12 changes: 10 additions & 2 deletions .github/workflows/publish.yml
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,8 @@ jobs:
("CLI package", "packages/fensu-cli/pyproject.toml"),
("CLI crate", "crates/fensu-cli/Cargo.toml"),
("policy crate", "crates/fensu-policy/Cargo.toml"),
("facts crate", "crates/fensu-facts/Cargo.toml"),
("native crate", "crates/fensu-native/Cargo.toml"),
]:
with Path(path).open("rb") as file:
document = tomllib.load(file)
Expand Down Expand Up @@ -197,6 +199,9 @@ jobs:
- uses: dtolnay/rust-toolchain@master
with:
toolchain: 1.85.0
- uses: dtolnay/rust-toolchain@master
with:
toolchain: 1.95.0
- name: Refresh released workspace versions in lockfile
run: cargo +1.85.0 metadata --format-version 1 > /dev/null
- name: Verify shared crate packages
Expand All @@ -210,6 +215,7 @@ jobs:
run: |
publish_if_missing() {
package="$1"
toolchain="$2"
version="$(cargo metadata --no-deps --format-version 1 \
| jq -r --arg package "${package}" '.packages[] | select(.name == $package) | .version')"
status="$(curl --retry 3 --silent --show-error --output /dev/null \
Expand All @@ -220,7 +226,7 @@ jobs:
echo "${package} ${version} is already published"
;;
404)
cargo publish --locked -p "${package}"
cargo "+${toolchain}" publish --locked -p "${package}"
;;
*)
echo "Could not determine ${package} ${version} publication status: HTTP ${status}" >&2
Expand All @@ -229,7 +235,9 @@ jobs:
esac
}

publish_if_missing fensu-policy
publish_if_missing fensu-facts 1.95.0
publish_if_missing fensu-native 1.95.0
publish_if_missing fensu-policy 1.85.0

pypi:
name: Publish to PyPI
Expand Down
3 changes: 3 additions & 0 deletions .github/workflows/verify.yml
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,9 @@ jobs:
steps:
- uses: actions/checkout@v4
- uses: dtolnay/rust-toolchain@stable
- uses: dtolnay/rust-toolchain@master
with:
toolchain: 1.95.0
- uses: dtolnay/rust-toolchain@master
with:
toolchain: 1.85.0
Expand Down
6 changes: 6 additions & 0 deletions .github/workflows/version-guard.yml
Original file line number Diff line number Diff line change
Expand Up @@ -39,13 +39,19 @@ jobs:
cli_crate_version = tomllib.load(cargo)["package"]["version"]
with Path("crates/fensu-policy/Cargo.toml").open("rb") as cargo:
policy_crate_version = tomllib.load(cargo)["package"]["version"]
with Path("crates/fensu-facts/Cargo.toml").open("rb") as cargo:
facts_crate_version = tomllib.load(cargo)["package"]["version"]
with Path("crates/fensu-native/Cargo.toml").open("rb") as cargo:
native_crate_version = tomllib.load(cargo)["package"]["version"]

versions = {
version,
cli_dependency,
cli_package_version,
cli_crate_version,
policy_crate_version,
facts_crate_version,
native_crate_version,
}
if len(versions) != 1:
print(f"Fensu package versions are not locked: {sorted(versions)}", file=sys.stderr)
Expand Down
5 changes: 3 additions & 2 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 2 additions & 1 deletion Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ command-group = "=5.0.1"
dunce = "1.0"
encoding_rs = "0.8"
flate2 = { version = "1", default-features = false, features = ["zlib"] }
get-size2 = { version = "=0.10.1", default-features = false }
globset = "=0.4.19"
hex = "0.4"
ignore = "0.4"
Expand All @@ -43,7 +44,7 @@ rusqlite = { version = "0.38", features = ["bundled"] }
serde_json = { version = "1", features = ["preserve_order"] }
serde = { version = "1", features = ["derive"] }
sha2 = "0.10"
fensu-facts = { path = "crates/fensu-facts", default-features = false }
fensu-facts = { path = "crates/fensu-facts", version = "0.29.3", default-features = false } # x-release-please-version
fensu-native = { path = "crates/fensu-native", default-features = false }
fensu-policy = { path = "crates/fensu-policy" }
fensu-rust = { path = "crates/fensu-rust" }
Expand Down
21 changes: 7 additions & 14 deletions crates/fensu-cli/src/check/_helpers/evaluation.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,10 @@ use std::collections::{HashMap, HashSet};
use std::path::Path;

use fensu_facts::extension::models::ProgramHandle;
use fensu_native::rules::main::evaluate_core_rules::evaluate_core_rules;
use fensu_native::rules::main::plan_core_rule_queries::plan_core_rule_queries;
use fensu_native::rules::main::plan_execution_owners::plan_execution_owners;
use fensu_native::rules::models::{NativeExecutionRule, NativeExecutionTarget, NativeRuleContext};
use fensu_native::evaluate_core_rules;
use fensu_native::plan_core_rule_queries;
use fensu_native::plan_execution_owners;
use fensu_native::{NativeExecutionRule, NativeExecutionTarget, NativeRuleContext};
use fensu_policy::lifecycle::models::FindingSeverity;
use rayon::iter::{IndexedParallelIterator, IntoParallelRefIterator, ParallelIterator};

Expand Down Expand Up @@ -169,11 +169,7 @@ pub(crate) fn evaluate(request: EvaluationRequest<'_>) -> Result<TargetEvaluatio
.iter()
.map(|code| (*code).to_owned())
.collect::<Vec<_>>();
for row in fensu_native::rules::main::evaluate_dead_code::evaluate_dead_code(
&codes,
&dead_code_context,
&project,
) {
for row in fensu_native::evaluate_dead_code(&codes, &dead_code_context, &project) {
let metadata = rule_metadata(&row.code)?
.ok_or_else(|| format!("Unknown native rule code: {}", row.code))?;
faults.push(Fault {
Expand Down Expand Up @@ -239,10 +235,7 @@ pub(crate) fn evaluate(request: EvaluationRequest<'_>) -> Result<TargetEvaluatio
})
}

fn dead_code_context(
config: &Config,
project_root: &Path,
) -> fensu_native::rules::models::NativeDeadCodeContext {
fn dead_code_context(config: &Config, project_root: &Path) -> fensu_native::NativeDeadCodeContext {
const PYPROJECT_SOURCE: &str = "pyproject";
if !config.dead_code.enabled {
return Default::default();
Expand All @@ -256,7 +249,7 @@ fn dead_code_context(
} else {
"fensu.toml"
};
fensu_native::rules::models::NativeDeadCodeContext {
fensu_native::NativeDeadCodeContext {
enabled: config.dead_code.enabled,
roots: config
.dead_code
Expand Down
4 changes: 2 additions & 2 deletions crates/fensu-cli/src/check/_helpers/project.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ use std::fs;
use std::path::{Component, Path, PathBuf};

use fensu_facts::extension::models::ProgramHandle;
use fensu_native::rules::models::{NativeProjectModule, NativeProjectPlane};
use fensu_native::{NativeProjectModule, NativeProjectPlane};
use globset::GlobBuilder;
use serde_json::Value;
use walkdir::DirEntry;
Expand Down Expand Up @@ -215,7 +215,7 @@ fn collect_entrypoint_values<'a>(value: &'a toml::Value, mut values: Vec<&'a str

pub(crate) fn observe(
root: &Path,
plans: &[fensu_native::rules::models::NativeProjectQuery],
plans: &[fensu_native::NativeProjectQuery],
programs: &HashMap<&str, &ProgramHandle>,
modules: &HashMap<String, &ProgramHandle>,
) -> Result<HashMap<String, Vec<String>>, String> {
Expand Down
2 changes: 1 addition & 1 deletion crates/fensu-cli/src/configuration/_helpers/parsing.rs
Original file line number Diff line number Diff line change
Expand Up @@ -148,7 +148,7 @@ pub(crate) fn parse_dead_code(
value: Option<&toml::Value>,
) -> Result<crate::models::DeadCodeConfig, String> {
let value = serde_json::to_value(value).map_err(|error| error.to_string())?;
let (enabled, roots) = fensu_native::rules::main::parse_dead_code::parse_dead_code(&value)?;
let (enabled, roots) = fensu_native::parse_dead_code(&value)?;
Ok(crate::models::DeadCodeConfig {
enabled,
roots: roots
Expand Down
11 changes: 9 additions & 2 deletions crates/fensu-facts/Cargo.toml
Original file line number Diff line number Diff line change
@@ -1,9 +1,14 @@
[package]
name = "fensu-facts"
version = "0.1.0"
version = "0.29.3" # x-release-please-version
edition.workspace = true
license.workspace = true
publish = false
publish = true
description = "Python source facts for Fensu: parsed modules, imports, symbols and file inventory"
repository = "https://github.com/sqlstack-io/fensu"
readme = "README.md"
keywords = ["lint", "architecture", "python", "static-analysis"]
categories = ["development-tools"]
rust-version.workspace = true

[lints]
Expand All @@ -16,6 +21,8 @@ crate-type = ["rlib"]

[dependencies]
dunce.workspace = true
# Not used directly: ruff_python_ast 0.0.4 needs a get-size2 built on compact_str 0.9, and 0.10.2 moved to 0.10.
get-size2.workspace = true
globset.workspace = true
hex.workspace = true
pyo3 = { workspace = true, optional = true }
Expand Down
11 changes: 11 additions & 0 deletions crates/fensu-facts/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
# fensu-facts

Python source facts for [Fensu](https://github.com/sqlstack-io/fensu): parse a module once into a `ProgramHandle`, then read its fact rows.

```toml
fensu-facts = { version = "0.30", default-features = false }
```

`default-features = false` leaves out the Python bindings.

The public API is `ProgramHandle` (built with `ProgramHandle::parse_source(source, major, minor)`), `ParseFailure`, and the row types in `facts`. Modules hidden from the documentation are shared with the rest of Fensu's workspace and may change in any release.
5 changes: 5 additions & 0 deletions crates/fensu-facts/src/extension/models.rs
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,11 @@ pub struct ProgramHandle {
}

impl ProgramHandle {
/// Parse `source` as a Python `major.minor` module.
pub fn parse_source(source: &str, major: u8, minor: u8) -> Result<Self, ParseFailure> {
Self::parse(source, PythonVersion { major, minor })
}

pub(crate) fn parse(source: &str, version: PythonVersion) -> Result<Self, ParseFailure> {
let parsed = parse_strict(source, version)?;
Ok(Self {
Expand Down
2 changes: 2 additions & 0 deletions crates/fensu-facts/src/facts/mod.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
//! CPython-shaped fact extraction over the ruff syntax tree.

#[doc(hidden)]
pub mod _helpers;
#[doc(hidden)]
pub mod main;
pub mod models;
pub mod types;
14 changes: 12 additions & 2 deletions crates/fensu-facts/src/lib.rs
Original file line number Diff line number Diff line change
@@ -1,12 +1,22 @@
//! Native fact-extraction core for fensu.
//! Python source facts for Fensu: [`ProgramHandle`] parses a module once and serves its fact rows.
#![forbid(unsafe_code)]

#[doc(hidden)]
pub mod constants;
#[doc(hidden)]
pub mod dead_code;
#[doc(hidden)]
pub mod extension;
pub mod facts;
#[doc(hidden)]
pub mod mapping;
#[doc(hidden)]
pub mod parsing;
#[doc(hidden)]
pub mod positions;
#[doc(hidden)]
pub mod snapshot;
pub mod syntax;
pub(crate) mod syntax;

pub use extension::models::ProgramHandle;
pub use parsing::models::ParseFailure;
2 changes: 1 addition & 1 deletion crates/fensu-facts/src/syntax/main/breadth_first_from.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
//! Walk breadth first from one node.

use crate::syntax::types::ShapeNode;
pub fn breadth_first_from(root: ShapeNode<'_>) -> Vec<ShapeNode<'_>> {
pub(crate) fn breadth_first_from(root: ShapeNode<'_>) -> Vec<ShapeNode<'_>> {
crate::syntax::_helpers::breadth::breadth_first_from(root)
}
2 changes: 1 addition & 1 deletion crates/fensu-facts/src/syntax/main/breadth_first_nodes.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,6 @@
use ruff_python_ast::ModModule;

use crate::syntax::types::ShapeNode;
pub fn breadth_first_nodes(module: &ModModule) -> Vec<ShapeNode<'_>> {
pub(crate) fn breadth_first_nodes(module: &ModModule) -> Vec<ShapeNode<'_>> {
crate::syntax::_helpers::breadth::breadth_first_nodes(module)
}
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@
use ruff_python_ast::ModModule;

use crate::syntax::types::ShapeNode;
pub fn breadth_first_with_parents(module: &ModModule) -> (Vec<ShapeNode<'_>>, Vec<Option<usize>>) {
pub(crate) fn breadth_first_with_parents(
module: &ModModule,
) -> (Vec<ShapeNode<'_>>, Vec<Option<usize>>) {
crate::syntax::_helpers::breadth::breadth_first_with_parents(module)
}
2 changes: 1 addition & 1 deletion crates/fensu-facts/src/syntax/main/children.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,6 @@
use crate::syntax::_helpers::children::ChildOutput;
use crate::syntax::types::ShapeNode;

pub fn children<'a>(node: &ShapeNode<'a>, out: &mut Vec<ShapeNode<'a>>) {
pub(crate) fn children<'a>(node: &ShapeNode<'a>, out: &mut Vec<ShapeNode<'a>>) {
ChildOutput { out }.collect(node);
}
2 changes: 1 addition & 1 deletion crates/fensu-facts/src/syntax/main/kind_name.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
//! Name the syntax kind of a node.

use crate::syntax::types::ShapeNode;
pub fn kind_name(node: &ShapeNode<'_>) -> &'static str {
pub(crate) fn kind_name(node: &ShapeNode<'_>) -> &'static str {
crate::syntax::_helpers::nodes::kind_name(node)
}
18 changes: 9 additions & 9 deletions crates/fensu-facts/src/syntax/main/mod.rs
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
//! Syntax entries.

pub mod breadth_first_from;
pub mod breadth_first_nodes;
pub mod breadth_first_with_parents;
pub mod children;
pub mod kind_name;
pub mod push_clause_orelse;
pub mod span;
pub mod start_of;
pub mod strict_reference_parts;
pub(crate) mod breadth_first_from;
pub(crate) mod breadth_first_nodes;
pub(crate) mod breadth_first_with_parents;
pub(crate) mod children;
pub(crate) mod kind_name;
pub(crate) mod push_clause_orelse;
pub(crate) mod span;
pub(crate) mod start_of;
pub(crate) mod strict_reference_parts;
2 changes: 1 addition & 1 deletion crates/fensu-facts/src/syntax/main/push_clause_orelse.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,6 @@ use ruff_python_ast::ElifElseClause;
use crate::syntax::_helpers::statement_children::ClauseOutput;
use crate::syntax::types::ShapeNode;

pub fn push_clause_orelse<'a>(clauses: &'a [ElifElseClause], out: &mut Vec<ShapeNode<'a>>) {
pub(crate) fn push_clause_orelse<'a>(clauses: &'a [ElifElseClause], out: &mut Vec<ShapeNode<'a>>) {
ClauseOutput { out }.collect(clauses);
}
Loading
Loading