log-frequency.py

#!/usr/bin/env python3
"""Count frequent log messages or regular-expression capture groups."""

import argparse
import re
import sys
from collections import Counter
from pathlib import Path


def parse_args():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("files", nargs="*", type=Path, help="Log files; stdin when omitted")
    parser.add_argument("--regex", help="Only matching lines; first capture group becomes the key")
    parser.add_argument("--top", type=int, default=20)
    parser.add_argument("--normalize-numbers", action="store_true")
    parser.add_argument("--ignore-case", action="store_true")
    return parser.parse_args()


def lines(files):
    if not files:
        yield from sys.stdin
        return
    for path in files:
        with path.open(encoding="utf-8", errors="replace") as handle:
            yield from handle


def main():
    args = parse_args()
    if args.top < 1:
        raise SystemExit("--top must be positive")
    pattern = re.compile(args.regex, re.IGNORECASE if args.ignore_case else 0) if args.regex else None
    counts = Counter()
    for line in lines(args.files):
        key = line.strip()
        if pattern:
            match = pattern.search(key)
            if not match:
                continue
            key = match.group(1) if match.groups() else match.group(0)
        if args.normalize_numbers:
            key = re.sub(r"\b\d+\b", "<N>", key)
        if args.ignore_case:
            key = key.lower()
        if key:
            counts[key] += 1
    for key, count in counts.most_common(args.top):
        print(f"{count}\t{key}")


if __name__ == "__main__":
    main()