diff --git a/CHANGELOG.md b/CHANGELOG.md index 874952e3..73090287 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ --- 5.0 --- +* issue#440: Remember only a successful table creation (not updates of existing tables) when avoiding repeated schema operations, so schema drift on an existing table, such as a narrowed owner column, is still detected and repaired within the same request +* issue#451: Treat the stale hostField value 'host_id' (shipped in config_local.php.dist since 2022) as the syslog_incoming 'host' column, and correct config_local.php.dist, so the device-rule mute/maintenance filter and host alerts no longer fail with "Unknown column 'syslog_incoming.host_id'" during alert processing +* issue#452, issue#453: Resolve remote collector hostnames from the Cacti poller table over the Cacti connection instead of joining it on the Syslog connection, and tolerate a failed query, so the Syslog Status and Dashboard tabs no longer throw a TypeError that disabled the plugin when the Syslog tables live in a separate database + * issue#440: Avoid repeating successful schema operations and seed the default dashboard after schema setup, preventing duplicate table creation during installation and upgrade * feature: Add an install-time timestamp precision choice for the logtime column (issue #325). New MariaDB installs default to millisecond TIMESTAMP(3), which preserves sub-second ordering and still RANGE partition-prunes; MySQL installs are fixed to second precision because MySQL cannot prune a fractional partitioned column, and an explicit millisecond request on MySQL is rejected. Upgrades never migrate the schema automatically; switch an existing MariaDB install with the destructive cli/updatedb_convert_ms.php utility during a maintenance window. README documents forwarder configuration (rsyslog, Cisco, Juniper, Arista, syslog-ng) for delivering sub-second timestamps diff --git a/config_local.php.dist b/config_local.php.dist index bb8da09d..a6f3eab1 100644 --- a/config_local.php.dist +++ b/config_local.php.dist @@ -106,6 +106,6 @@ $syslog_install_options['id'] = 'syslog'; $syslog_incoming_config['timeField'] = 'logtime'; $syslog_incoming_config['priorityField'] = 'priority_id'; $syslog_incoming_config['facilityField'] = 'facility_id'; -$syslog_incoming_config['hostField'] = 'host_id'; +$syslog_incoming_config['hostField'] = 'host'; $syslog_incoming_config['textField'] = 'message'; $syslog_incoming_config['id'] = 'seq'; diff --git a/includes/functions.php b/includes/functions.php index 89ed9a74..0e1ccd30 100644 --- a/includes/functions.php +++ b/includes/functions.php @@ -3388,7 +3388,7 @@ function syslog_get_removal_rule_fields($table = 'syslog_incoming', $prefix = '' return [ 'message' => ['column' => $column($syslog_incoming_config['textField']), 'operators' => ['contains', 'begins', 'ends', '=', '!=']], - 'host' => ['column' => $column($syslog_incoming_config['hostField']), 'operators' => ['contains', 'begins', 'ends', '=', '!=']], + 'host' => ['column' => $column(syslog_incoming_host_field()), 'operators' => ['contains', 'begins', 'ends', '=', '!=']], 'program' => ['column' => $column($syslog_incoming_config['programField']), 'operators' => ['contains', 'begins', 'ends', '=', '!=']], 'facility_id' => ['column' => $column($syslog_incoming_config['facilityField']), 'operators' => ['=', '!='], 'type' => 'integer'], 'priority_id' => ['column' => $column($syslog_incoming_config['priorityField']), 'operators' => ['=', '!='], 'type' => 'integer'] @@ -3715,7 +3715,7 @@ function syslog_rule_preview(array $rule, string $rule_type = 'alert', int $rows $incoming_fields = [ 'facility' => $syslog_incoming_config['facilityField'], - 'host' => $syslog_incoming_config['hostField'], + 'host' => syslog_incoming_host_field(), 'program' => $syslog_incoming_config['programField'], ]; @@ -5279,6 +5279,23 @@ function syslog_alert_maintenance_is_active(array $alert): bool { || syslog_alert_schedule_is_active(syslog_alert_maintenance_window((string) read_config_option('syslog_alert_maintenance_days'), (string) read_config_option('syslog_alert_maintenance_start'), (string) read_config_option('syslog_alert_maintenance_end'))); } +/** + * Return the column that holds the hostname in the syslog_incoming table. + * + * Releases before 4.11 shipped config_local.php.dist with hostField set to + * 'host_id', which is not a syslog_incoming column; the staging table has + * always stored the hostname in 'host'. Treat that stale value as 'host'. + * + * @return string + */ +function syslog_incoming_host_field(): string { + global $syslog_incoming_config; + + $field = $syslog_incoming_config['hostField'] ?? 'host'; + + return ($field === '' || $field === 'host_id') ? 'host' : (string) $field; +} + /** * Limit an alert query by active device-wide handling rules. * @@ -5290,7 +5307,7 @@ function syslog_alert_maintenance_is_active(array $alert): bool { */ function syslog_device_rule_sql(bool $maintenance_active): array { global $syslogdb_default, $syslog_incoming_config; - $host_field = $syslog_incoming_config['hostField'] ?? 'host'; + $host_field = syslog_incoming_host_field(); $priority_field = $syslog_incoming_config['priorityField'] ?? 'priority_id'; $now = time(); // Qualify the outer staging columns explicitly: an unqualified `host` @@ -6241,7 +6258,7 @@ function syslog_get_alert_sql(&$alert, $max_seq, $processing_boundary = true, $c } $fields = [ 'message' => ['column' => '`' . $syslog_incoming_config['textField'] . '`', 'operators' => ['contains', 'begins', 'ends', '=', '!=']], - 'host' => ['column' => '`' . $syslog_incoming_config['hostField'] . '`', 'operators' => ['contains', 'begins', 'ends', '=', '!=']], + 'host' => ['column' => '`' . syslog_incoming_host_field() . '`', 'operators' => ['contains', 'begins', 'ends', '=', '!=']], 'program' => ['column' => '`' . $syslog_incoming_config['programField'] . '`', 'operators' => ['contains', 'begins', 'ends', '=', '!=']], 'facility_id' => ['column' => '`' . $syslog_incoming_config['facilityField'] . '`', 'operators' => ['=', '!='], 'type' => 'integer'], 'priority_id' => ['column' => '`' . $syslog_incoming_config['priorityField'] . '`', 'operators' => ['=', '!='], 'type' => 'integer'] @@ -6265,7 +6282,7 @@ function syslog_get_alert_sql(&$alert, $max_seq, $processing_boundary = true, $c case 'messagee': return ['sql' => "`{$syslog_incoming_config['textField']}` LIKE ?", 'params' => ['%' . $alert['message']]]; case 'host': - return ['sql' => "`{$syslog_incoming_config['hostField']}` = ?", 'params' => [$alert['message']]]; + return ['sql' => '`' . syslog_incoming_host_field() . '` = ?', 'params' => [$alert['message']]]; case 'program': return ['sql' => "`{$syslog_incoming_config['programField']}` = ?", 'params' => [$alert['message']]]; case 'sql': @@ -6281,7 +6298,7 @@ function syslog_get_alert_sql(&$alert, $max_seq, $processing_boundary = true, $c } $fields = [ 'message' => ['column' => '`' . $syslog_incoming_config['textField'] . '`', 'operators' => ['contains', 'begins', 'ends', '=', '!=']], - 'host' => ['column' => '`' . $syslog_incoming_config['hostField'] . '`', 'operators' => ['contains', 'begins', 'ends', '=', '!=']], + 'host' => ['column' => '`' . syslog_incoming_host_field() . '`', 'operators' => ['contains', 'begins', 'ends', '=', '!=']], 'program' => ['column' => '`' . $syslog_incoming_config['programField'] . '`', 'operators' => ['contains', 'begins', 'ends', '=', '!=']], 'facility_id' => ['column' => '`' . $syslog_incoming_config['facilityField'] . '`', 'operators' => ['=', '!='], 'type' => 'integer'], 'priority_id' => ['column' => '`' . $syslog_incoming_config['priorityField'] . '`', 'operators' => ['=', '!='], 'type' => 'integer'] @@ -6347,7 +6364,7 @@ function syslog_get_alert_sql(&$alert, $max_seq, $processing_boundary = true, $c } elseif ($alert['type'] == 'host') { $sql = "SELECT * FROM `$syslogdb_default`.`syslog_incoming` - WHERE `{$syslog_incoming_config['hostField']}` = ? + WHERE `" . syslog_incoming_host_field() . "` = ? AND `status` = 1 AND `seq` <= ?"; @@ -7039,12 +7056,33 @@ function syslog_replication_collector_status(): array { return []; } - return syslog_db_fetch_assoc("SELECT rc.source_poller_id, p.hostname, + $collectors = syslog_db_fetch_assoc("SELECT rc.source_poller_id, rc.last_batch_count, UNIX_TIMESTAMP(rc.last_received) AS last_received FROM `$syslogdb_default`.`syslog_replication_collectors` AS rc - LEFT JOIN poller AS p - ON p.id = rc.source_poller_id ORDER BY rc.last_received DESC, rc.source_poller_id ASC", false); + + if (!is_array($collectors) || !cacti_sizeof($collectors)) { + return []; + } + + // The poller table lives in the Cacti database, which may differ from the + // dedicated Syslog database, so resolve hostnames over the Cacti connection. + $poller_ids = array_values(array_unique(array_map('intval', array_column($collectors, 'source_poller_id')))); + $hostnames = []; + $pollers = db_fetch_assoc_prepared('SELECT id, hostname FROM poller WHERE id IN (' . implode(',', array_fill(0, cacti_sizeof($poller_ids), '?')) . ')', $poller_ids, false); + + if (is_array($pollers)) { + foreach ($pollers as $poller) { + $hostnames[(int) $poller['id']] = $poller['hostname']; + } + } + + foreach ($collectors as &$collector) { + $collector['hostname'] = $hostnames[(int) $collector['source_poller_id']] ?? null; + } + unset($collector); + + return $collectors; } /** Conservative online delivery limit. Payloads may contain 2KiB messages. */ diff --git a/includes/schema.php b/includes/schema.php index f2d537c7..4a704b13 100644 --- a/includes/schema.php +++ b/includes/schema.php @@ -60,26 +60,31 @@ function syslog_db_update_table($table, $data, $removecolumns = false, $log = tr static $applied = []; - // Core caches missing tables even after CREATE. Do not repeat an identical - // successful schema operation in this request (connect and upgrade can overlap). + // Core caches missing tables even after CREATE, so a second db_update_table() + // in this request would try to create the table again. Only a successful + // creation is remembered; updates of existing tables always run so schema + // drift is still repaired (connect and upgrade can overlap). $connection = is_object($db_conn) ? $db_conn : $syslog_cnn; $key = (is_object($connection) ? spl_object_hash($connection) : 'default') . ':' . $table; $definition = [$connection, $data, $removecolumns]; + $exists = syslog_db_table_exists($table, false); - if (isset($applied[$key]) && $applied[$key] === $definition && syslog_db_table_exists($table, false)) { + if ($exists && isset($applied[$key]) && $applied[$key] === $definition) { return true; } if (isset($data['primary']) && is_array($data['primary']) && !syslog_cacti_db_array_primary_safe() - && !syslog_db_table_exists($table, false)) { + && !$exists) { $data['primary'] = implode('`,`', $data['primary']); } $result = db_update_table($table, $data, $removecolumns, $log, $db_conn); - if ($result) { + if ($result && !$exists) { $applied[$key] = $definition; + } else { + unset($applied[$key]); } return $result; diff --git a/tests/Unit/DeviceRulePolicyTest.php b/tests/Unit/DeviceRulePolicyTest.php index c6eb8eb8..4e10eeb5 100644 --- a/tests/Unit/DeviceRulePolicyTest.php +++ b/tests/Unit/DeviceRulePolicyTest.php @@ -56,6 +56,21 @@ ->toContain("syslog_replace_data(\$table, \$rows, \$data['rcnn_id'], \$syslog_cnn)"); }); +it('maps the legacy host_id hostField to the incoming host column', function () { + syslog_load_plugin_source('includes/functions.php'); + $GLOBALS['syslogdb_default'] = 'syslog'; + $GLOBALS['syslog_incoming_config'] = ['hostField' => 'host_id', 'priorityField' => 'priority_id']; + + expect(syslog_incoming_host_field())->toBe('host'); + + $filter = syslog_device_rule_sql(true); + expect($filter['sql'])->toContain('dr.host = `syslog_incoming`.`host`') + ->not->toContain('`host_id`'); + + $GLOBALS['syslog_incoming_config'] = ['hostField' => 'src_host']; + expect(syslog_incoming_host_field())->toBe('src_host'); +}); + it('correlates device suppression and maintenance exceptions with the incoming message host', function () { syslog_load_plugin_source('includes/functions.php'); $GLOBALS['syslogdb_default'] = 'syslog'; diff --git a/tests/Unit/ReplicationDeliveryTest.php b/tests/Unit/ReplicationDeliveryTest.php index 42599680..941d8578 100644 --- a/tests/Unit/ReplicationDeliveryTest.php +++ b/tests/Unit/ReplicationDeliveryTest.php @@ -53,17 +53,45 @@ $GLOBALS['config'] = ['poller_id' => 1]; $GLOBALS['syslogdb_default'] = 'cacti'; $sql = ''; + $poller_sql = ''; test_override('syslog_db_table_exists', fn ($table) => $table === 'syslog_replication_collectors'); test_override('syslog_db_fetch_assoc', function ($statement) use (&$sql) { $sql = $statement; return [[ - 'source_poller_id' => 2, 'hostname' => 'remote-poller.example.com', 'last_batch_count' => 100, 'last_received' => 1758801600, + 'source_poller_id' => 2, 'last_batch_count' => 100, 'last_received' => 1758801600, ]]; }); + test_override('db_fetch_assoc_prepared', function ($statement) use (&$poller_sql) { $poller_sql = $statement; return [['id' => 2, 'hostname' => 'remote-poller.example.com']]; }); syslog_load_plugin_source('includes/functions.php'); - expect(syslog_replication_collector_status())->toHaveCount(1) + $collectors = syslog_replication_collector_status(); + + expect($collectors)->toHaveCount(1) + ->and($collectors[0]['hostname'])->toBe('remote-poller.example.com') ->and($sql)->toContain('syslog_replication_collectors') - ->toContain('LEFT JOIN poller AS p') - ->toContain('p.hostname') - ->not->toContain('`.`syslog`'); + ->not->toContain('poller AS p') + ->not->toContain('`.`syslog`') + ->and($poller_sql)->toContain('FROM poller'); +}); + +it('returns no collectors instead of a TypeError when the receipt query fails', function () { + $GLOBALS['config'] = ['poller_id' => 1]; + $GLOBALS['syslogdb_default'] = 'syslog_db'; + test_override('syslog_db_table_exists', fn () => true); + test_override('syslog_db_fetch_assoc', fn () => false); + syslog_load_plugin_source('includes/functions.php'); + + expect(syslog_replication_collector_status())->toBe([]); +}); + +it('keeps collectors listed when the poller hostname lookup fails', function () { + $GLOBALS['config'] = ['poller_id' => 1]; + $GLOBALS['syslogdb_default'] = 'syslog_db'; + test_override('syslog_db_table_exists', fn () => true); + test_override('syslog_db_fetch_assoc', fn () => [['source_poller_id' => 3, 'last_batch_count' => 5, 'last_received' => 1]]); + test_override('db_fetch_assoc_prepared', fn () => false); + syslog_load_plugin_source('includes/functions.php'); + + $collectors = syslog_replication_collector_status(); + + expect($collectors)->toHaveCount(1)->and($collectors[0]['hostname'])->toBeNull(); }); it('treats an unset poller ID as the Main Collector for receipt telemetry', function () { diff --git a/tests/Unit/SchemaPrimaryKeyCompatTest.php b/tests/Unit/SchemaPrimaryKeyCompatTest.php index 2a7111ed..fca4ee76 100644 --- a/tests/Unit/SchemaPrimaryKeyCompatTest.php +++ b/tests/Unit/SchemaPrimaryKeyCompatTest.php @@ -118,3 +118,24 @@ function schema_primary_capture(string $cacti_version, bool $table_exists): arra expect(syslog_db_update_table('syslog_repeat_probe', $definition, false, true, $connection))->toBeTrue(); expect($calls)->toBe(4); }); + + +it('always reaches db_update_table() for repeated updates of an existing table and propagates failures', function () { + syslog_load_plugin_source('includes/schema.php'); + $GLOBALS['config']['cacti_version'] = '1.2.32'; + $connection = new stdClass(); + $results = [true, true, false]; + $calls = 0; + test_override('syslog_db_table_exists', fn () => true); + test_override('db_update_table', function () use (&$calls, &$results) { + $calls++; + return array_shift($results); + }); + $definition = ['columns' => [['name' => 'user', 'type' => 'varchar(50)']], 'primary' => ['id']]; + + expect(syslog_db_update_table('syslog_existing_probe', $definition, false, true, $connection))->toBeTrue(); + expect(syslog_db_update_table('syslog_existing_probe', $definition, false, true, $connection))->toBeTrue(); + expect($calls)->toBe(2); + expect(syslog_db_update_table('syslog_existing_probe', $definition, false, true, $connection))->toBeFalse(); + expect($calls)->toBe(3); +}); \ No newline at end of file