Repository navigation
Expand file tree
/
Copy pathserver.py
More file actions
1290 lines (1061 loc) · 44.2 KB
/
Copy pathserver.py
File metadata and controls
1290 lines (1061 loc) · 44.2 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
"""
Token Optimization MCP Server - Custom context & tool optimization for Claude Code.
Consolidates 8 scattered optimization scripts (4,500+ lines) into a unified
MCP server. Provides pre-execution tool optimization, AST code navigation,
smart file analysis, context deduplication, and optimization metrics.
THIS IS A CUSTOM SYSTEM - not a standard filesystem MCP. It provides:
- 60-85% overall token reduction
- AST-based code navigation (80-95% savings on code exploration)
- Pre-execution tool call interception and optimization
- Context deduplication across SRS/README/CLAUDE.md
- Smart file reading strategy recommendations
- Real-time optimization metrics and logging
Backend: Direct file I/O + AST parsing + regex-based code navigation
Transport: stdio
Tools (10):
optimize_tool_call, ast_navigate_code, smart_read_analyze,
deduplicate_context, dedup_estimate, context_budget_status,
get_optimization_stats, log_optimization, optimize_read_params,
optimize_grep_params
"""
import ast as ast_module
import hashlib
import inspect
import json
import logging
import os
import re
import sys
from datetime import datetime
from pathlib import Path
from typing import Annotated
from pydantic import Field
sys.path.insert(0, str(Path(__file__).resolve().parent))
from utils.path_resolver import get_data_dir, get_tool_opt_log
# mcp 2.0 renamed FastMCP to MCPServer and moved it to mcp.server.mcpserver.
# Both names are probed so this server runs under either major version; the
# API used below (tool decorator, run(transport=...)) is identical in both.
try:
from mcp.server.mcpserver import MCPServer
except ImportError: # mcp < 2.0
from mcp.server.fastmcp import FastMCP as MCPServer
try:
from mcp.types import ToolAnnotations
except ImportError: # very old SDK without the annotations model
ToolAnnotations = None
from base.response import to_json
from base.decorators import mcp_tool_handler
from base.persistence import JsonlAppender
mcp = MCPServer(
"token-optimizer",
instructions="Custom token & context optimization (60-85% reduction)"
)
_LOG = logging.getLogger("token_optimizer")
def _configure_logging():
"""Attach a stderr handler so log output never corrupts the stdio channel.
stdout carries newline-delimited JSON-RPC frames; anything else written
there breaks the transport, so all diagnostics go to stderr.
"""
if not _LOG.handlers:
handler = logging.StreamHandler(sys.stderr)
handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(name)s %(message)s"))
_LOG.addHandler(handler)
_LOG.propagate = False
level_name = os.environ.get("TOKEN_OPTIMIZER_LOG_LEVEL", "INFO").upper()
_LOG.setLevel(getattr(logging, level_name, logging.INFO))
_configure_logging()
def _log_event(level, event, **fields):
"""Emit one structured, single-line JSON log record.
Args:
level: Standard ``logging`` level constant.
event: Short machine-readable event name.
**fields: Additional key/value pairs to include in the record.
"""
payload = {"event": event}
payload.update(fields)
_LOG.log(level, json.dumps(payload, default=str))
_TOOL_KWARGS = set(inspect.signature(mcp.tool).parameters)
def _tool(**kwargs):
"""Register an MCP tool, dropping kwargs the installed SDK does not accept.
``annotations`` and ``structured_output`` were added to FastMCP at
different points and requirements.txt only pins ``mcp>=1.0.0``, so
unsupported keywords are filtered rather than raising at import time.
Args:
**kwargs: Keyword arguments for the underlying ``mcp.tool`` decorator.
Returns:
The decorator returned by ``mcp.tool``.
"""
supported = {key: value for key, value in kwargs.items() if key in _TOOL_KWARGS}
return mcp.tool(**supported)
def _annotations(title, read_only, destructive, idempotent, open_world=False):
"""Build a ``ToolAnnotations`` object, or None on an SDK without the model.
An omitted annotation set is read by the specification as the least-safe
possible declaration, so every tool here declares all four hints.
Args:
title: Human-readable tool title.
read_only: True when the tool performs no writes.
destructive: True when the tool's effect is irreversible.
idempotent: True when repeat calls with identical arguments have the
same cumulative effect as a single call.
open_world: True when the tool reaches an external or open-ended system.
Returns:
A ``ToolAnnotations`` instance, or None when unavailable.
"""
if ToolAnnotations is None:
return None
return ToolAnnotations(
title=title,
readOnlyHint=read_only,
destructiveHint=destructive,
idempotentHint=idempotent,
openWorldHint=open_world,
)
def _env_int(name, default):
"""Read a positive int from the environment, falling back on bad input.
Args:
name: Environment variable name.
default: Value used when unset, unparseable, or non-positive.
Returns:
The resolved int.
"""
raw = os.environ.get(name, "").strip()
if not raw:
return default
try:
value = int(raw)
except ValueError:
_log_event(logging.WARNING, "invalid_env_value", name=name, value=raw, using=default)
return default
if value <= 0:
_log_event(logging.WARNING, "non_positive_env_value", name=name, value=value, using=default)
return default
return value
# Paths
MEMORY_PATH = get_data_dir()
OPTIMIZATION_LOG = get_tool_opt_log()
LOGS_PATH = OPTIMIZATION_LOG.parent
# Structured logger for optimization events
_opt_logger = JsonlAppender(OPTIMIZATION_LOG)
CONTEXT_BUDGET_BYTES = 200 * 1024 # 200KB budget
# Track file access counts (in-process cache, bounded to avoid unbounded growth)
_file_access_count = {}
_MAX_TRACKED_FILES = _env_int("TOKEN_OPTIMIZER_MAX_TRACKED_FILES", 5000)
# Bounds on inputs and scans
_MAX_CONTEXT_BYTES = _env_int("TOKEN_OPTIMIZER_MAX_CONTEXT_BYTES", 8 * 1024 * 1024)
_MAX_STATS_LINES = _env_int("TOKEN_OPTIMIZER_MAX_STATS_LINES", 200000)
_MAX_BUDGET_SCAN_FILES = _env_int("TOKEN_OPTIMIZER_MAX_BUDGET_SCAN_FILES", 50000)
# Dedup constants
_DEDUP_MIN_SAVINGS = 0.20 # 20% threshold
_DEDUP_PRIORITY = ["srs", "readme", "claude_md"]
def _count_lines(path):
"""Count newline-delimited lines in a file, closing the handle explicitly.
Args:
path: Path to the file to count.
Returns:
Number of lines in the file.
"""
with open(path, "rb") as handle:
return sum(1 for _ in handle)
def _record_access(file_path):
"""Increment the in-process access counter for a file path.
The counter map is cleared once it exceeds the configured cap so a
long-lived server cannot accumulate an unbounded number of path keys.
Args:
file_path: Path string used as the counter key.
Returns:
The updated access count for this path.
"""
if len(_file_access_count) >= _MAX_TRACKED_FILES and file_path not in _file_access_count:
_log_event(
logging.INFO,
"access_counter_reset",
tracked_files=len(_file_access_count),
cap=_MAX_TRACKED_FILES,
)
_file_access_count.clear()
_file_access_count[file_path] = _file_access_count.get(file_path, 0) + 1
return _file_access_count[file_path]
# =============================================================================
# TOOL 1: OPTIMIZE ANY TOOL CALL (Interceptor)
# =============================================================================
@_tool(
annotations=_annotations("Optimize a tool call", True, False, True),
structured_output=False,
)
@mcp_tool_handler
def optimize_tool_call(
tool_name: Annotated[str, Field(description="Tool name to optimize for: Read, Grep, Glob, Bash, Edit, or Write. Matched case-insensitively.")],
params: Annotated[str, Field(description="JSON object string of the parameters that would be passed to the tool. Invalid JSON is treated as an empty object.")] = "{}",
) -> str:
"""Intercept and optimize any Claude tool call before execution.
Applies tool-specific optimizations (head_limit, offset/limit, path
restriction, output_mode selection) and returns optimized parameters
with suggestions and estimated token savings.
Use this before issuing a large or exploratory tool call. It only inspects
files to size them; it never modifies anything.
"""
try:
p = json.loads(params)
except (json.JSONDecodeError, TypeError):
p = {}
tool = tool_name.lower()
optimized = dict(p)
suggestions = []
savings = 0
if tool == "read":
result = _optimize_read(p)
optimized, suggestions, savings = result
elif tool == "grep":
result = _optimize_grep(p)
optimized, suggestions, savings = result
elif tool == "glob":
result = _optimize_glob(p)
optimized, suggestions, savings = result
elif tool == "bash":
result = _optimize_bash(p)
optimized, suggestions, savings = result
elif tool == "edit":
result = _optimize_edit(p)
optimized, suggestions, savings = result
elif tool == "write":
result = _optimize_write(p)
optimized, suggestions, savings = result
# Log optimization
_log_opt(tool_name, p != optimized, savings, len(suggestions))
return to_json({
"success": True,
"tool": tool_name,
"original_params": p,
"optimized_params": optimized,
"suggestions": suggestions,
"token_savings_estimate": savings,
"was_optimized": p != optimized
})
def _optimize_read(p: dict):
"""Suggest offset/limit and alternative strategies for a Read call.
Args:
p: Original Read parameters.
Returns:
Tuple of (optimized params, suggestion list, estimated token savings).
"""
file_path = p.get("file_path", "")
offset = p.get("offset")
limit = p.get("limit")
optimized = dict(p)
suggestions = []
savings = 0
access_count = _record_access(file_path)
path = Path(file_path)
if path.exists() and path.is_file():
try:
line_count = _count_lines(path)
# Large file without offset/limit
if line_count > 500 and offset is None and limit is None:
optimized["offset"] = 0
optimized["limit"] = 200
suggestions.append(
f"Auto-optimized: {line_count} lines -> offset=0, limit=200"
)
savings = (line_count - 200) * 80 # ~80 tokens/line
# Medium file
elif line_count > 200 and offset is None and limit is None:
optimized["limit"] = line_count # Allow full read but note it
suggestions.append(
f"Medium file ({line_count} lines) - consider limit if exploring"
)
# Frequent access -> cache suggestion
if access_count >= 3:
suggestions.append(
f"Cache opportunity: accessed {access_count} times"
)
# Code file -> suggest AST navigation instead
ext = path.suffix.lower()
if ext in (".java", ".py", ".ts", ".tsx") and line_count > 300:
suggestions.append(
"Consider ast_navigate_code for structure (80-95% savings)"
)
except OSError as exc:
_log_event(
logging.WARNING,
"read_optimization_probe_failed",
file_path=file_path,
error_type=type(exc).__name__,
error=str(exc)[:200],
)
suggestions.append(f"Could not size file for optimization: {exc}")
return optimized, suggestions, savings
def _optimize_grep(p: dict):
"""Enforce head_limit and output_mode defaults for a Grep call.
Args:
p: Original Grep parameters.
Returns:
Tuple of (optimized params, suggestion list, estimated token savings).
"""
pattern = p.get("pattern", "")
head_limit = p.get("head_limit")
output_mode = p.get("output_mode")
optimized = dict(p)
suggestions = []
savings = 0
# No head_limit -> enforce default
if head_limit is None:
optimized["head_limit"] = 100
suggestions.append("Auto-optimized: added head_limit=100")
savings += 1000
# No output_mode -> use files_with_matches for initial search
if output_mode is None:
optimized["output_mode"] = "files_with_matches"
suggestions.append("Auto-optimized: output_mode='files_with_matches' for initial search")
savings += 2000
# Very broad pattern
if len(pattern) < 3:
suggestions.append(f"Pattern '{pattern}' is very broad - consider more specific search")
return optimized, suggestions, savings
def _optimize_glob(p: dict):
"""Flag overly broad Glob patterns and missing path restrictions.
Args:
p: Original Glob parameters.
Returns:
Tuple of (optimized params, suggestion list, estimated token savings).
"""
pattern = p.get("pattern", "")
path = p.get("path")
optimized = dict(p)
suggestions = []
savings = 0
if path is None:
suggestions.append("Consider restricting path if service directory is known")
if "**/*" in pattern and "." not in pattern:
suggestions.append(f"Pattern '{pattern}' is very broad - add file extension filter")
return optimized, suggestions, savings
def _optimize_bash(p: dict):
"""Suggest output limiting and parallelization for a Bash command.
Args:
p: Original Bash parameters.
Returns:
Tuple of (optimized params, suggestion list, estimated token savings).
"""
command = p.get("command", "")
optimized = dict(p)
suggestions = []
savings = 0
# Structure exploration -> suggest tree pattern
structure_cmds = ["find", "ls -R", "dir /s"]
if any(cmd in command for cmd in structure_cmds):
suggestions.append(
"Tree Pattern: use 'find . -maxdepth 2 -type d | sort' for structure"
)
savings += 500
# Too many sequential commands
if "&&" in command:
parts = [p.strip() for p in command.split("&&")]
if len(parts) > 3:
suggestions.append(
f"{len(parts)} sequential commands - consider parallelizing independent ones"
)
# Missing output limiting
if "grep" in command and "| head" not in command and "--head" not in command:
suggestions.append("Add '| head -50' to limit grep output in bash")
savings += 300
return optimized, suggestions, savings
def _optimize_edit(p: dict):
"""Check that an Edit's old_string occurs exactly once in the target file.
Args:
p: Original Edit parameters.
Returns:
Tuple of (optimized params, suggestion list, estimated token savings).
"""
file_path = p.get("file_path", "")
old_string = p.get("old_string", "")
optimized = dict(p)
suggestions = []
savings = 0
path = Path(file_path)
if path.exists() and old_string:
try:
content = path.read_text(encoding="utf-8", errors="replace")
count = content.count(old_string)
if count > 1:
suggestions.append(
f"old_string appears {count} times - Edit will fail! "
"Provide more context or use replace_all=True"
)
elif count == 0:
suggestions.append("old_string NOT FOUND in file - check exact whitespace/indentation")
except OSError as exc:
_log_event(
logging.WARNING,
"edit_optimization_probe_failed",
file_path=file_path,
error_type=type(exc).__name__,
error=str(exc)[:200],
)
suggestions.append(f"Could not read file to verify old_string: {exc}")
return optimized, suggestions, savings
def _optimize_write(p: dict):
"""Warn when a Write target's parent directory does not exist.
Args:
p: Original Write parameters.
Returns:
Tuple of (optimized params, suggestion list, estimated token savings).
"""
file_path = p.get("file_path", "")
optimized = dict(p)
suggestions = []
savings = 0
path = Path(file_path)
if not path.parent.exists():
suggestions.append(f"Parent directory does not exist: {path.parent}")
return optimized, suggestions, savings
# =============================================================================
# TOOL 2: AST CODE NAVIGATION (80-95% savings)
# =============================================================================
@_tool(
annotations=_annotations("Navigate code structure", True, False, True),
structured_output=False,
)
@mcp_tool_handler
def ast_navigate_code(
file_path: Annotated[str, Field(description="Path to the code file to analyze. Supported extensions: .py (stdlib AST), .java, .ts, .tsx, .js (regex).")],
show_methods: Annotated[bool, Field(description="When true, include method and function signatures in the output. When false, only top-level declarations are returned.")] = False,
) -> str:
"""Extract code structure without reading full file content.
Uses AST parsing for Python, regex for Java/TypeScript/JavaScript.
Returns: package, imports, classes, interfaces, methods with line numbers.
Saves 80-95% tokens compared to reading the full file.
Use this to orient in an unfamiliar file before reading any of it. It is
read-only and does not follow imports into other files.
"""
path = Path(file_path)
if not path.exists():
return to_json({"success": False, "error": f"File not found: {file_path}"})
ext = path.suffix.lower()
content = path.read_text(encoding="utf-8", errors="ignore")
line_count = content.count("\n") + 1
if ext == ".java":
result = _navigate_java(content, show_methods)
elif ext in (".ts", ".tsx"):
result = _navigate_typescript(content, show_methods)
elif ext == ".py":
result = _navigate_python(file_path, content, show_methods)
elif ext == ".js":
result = _navigate_typescript(content, show_methods)
result["language"] = "javascript"
else:
return to_json({"success": False, "error": f"Unsupported: {ext}"})
result["file"] = file_path
result["total_lines"] = line_count
result["tokens_saved_estimate"] = line_count * 80 # ~80 tokens/line saved
result["success"] = True
return to_json(result)
def _navigate_java(content: str, show_methods: bool) -> dict:
package = re.search(r"package\s+([\w.]+);", content)
imports = re.findall(r"import\s+([\w.]+);", content)
classes = re.findall(r"(?:public\s+)?class\s+(\w+)", content)
interfaces = re.findall(r"(?:public\s+)?interface\s+(\w+)", content)
result = {
"language": "java",
"package": package.group(1) if package else None,
"imports": imports[:10],
"classes": classes,
"interfaces": interfaces,
}
if show_methods:
methods = re.findall(
r"(public|private|protected)\s+(?:static\s+)?(\w+)\s+(\w+)\s*\([^)]*\)",
content,
)
result["methods"] = [
{"visibility": m[0], "return_type": m[1], "name": m[2]} for m in methods
]
return result
def _navigate_typescript(content: str, show_methods: bool) -> dict:
imports = re.findall(r"import\s+.*?from\s+['\"]([^'\"]+)['\"]", content)
classes = re.findall(r"export\s+class\s+(\w+)", content)
interfaces = re.findall(r"export\s+interface\s+(\w+)", content)
functions = re.findall(r"export\s+function\s+(\w+)", content)
consts = re.findall(r"export\s+const\s+(\w+)", content)
result = {
"language": "typescript",
"imports": imports[:10],
"classes": classes,
"interfaces": interfaces,
"functions": functions,
"constants": consts,
}
if show_methods:
class_methods = re.findall(r"(\w+)\s*\([^)]*\)\s*:\s*(\w+)", content)
result["methods"] = [{"name": m[0], "return_type": m[1]} for m in class_methods]
return result
_PY_FUNCTION_NODES = (ast_module.FunctionDef, ast_module.AsyncFunctionDef)
def _navigate_python(file_path: str, content: str, show_methods: bool) -> dict:
"""Extract imports, classes, and module-level functions via the stdlib AST.
Functions are taken from the module body rather than a full tree walk, so
methods and closures are not misreported as module-level functions. Both
``def`` and ``async def`` are recognized.
Args:
file_path: Path used only for the syntax-error message.
content: Full source text.
show_methods: When true, list each class's method names.
Returns:
Dict with language, imports, classes, and functions.
"""
try:
tree = ast_module.parse(content, filename=file_path)
except SyntaxError as exc:
_log_event(
logging.INFO,
"python_parse_failed",
file_path=file_path,
lineno=exc.lineno,
error=str(exc.msg),
)
return {
"language": "python",
"error": "Syntax error in file at line %s: %s" % (exc.lineno, exc.msg),
}
imports = []
for node in ast_module.walk(tree):
if isinstance(node, ast_module.Import):
imports.extend(alias.name for alias in node.names)
elif isinstance(node, ast_module.ImportFrom) and node.module:
imports.append(node.module)
classes = []
functions = []
for node in tree.body:
if isinstance(node, ast_module.ClassDef):
cls = {"name": node.name, "line": node.lineno}
if show_methods:
cls["methods"] = [
member.name
for member in node.body
if isinstance(member, _PY_FUNCTION_NODES)
]
classes.append(cls)
elif isinstance(node, _PY_FUNCTION_NODES):
functions.append(
{
"name": node.name,
"line": node.lineno,
"is_async": isinstance(node, ast_module.AsyncFunctionDef),
}
)
return {
"language": "python",
"imports": imports[:10],
"classes": classes,
"functions": functions,
}
# =============================================================================
# TOOL 3: SMART FILE ANALYSIS
# =============================================================================
@_tool(
annotations=_annotations("Analyze read strategy", True, False, True),
structured_output=False,
)
@mcp_tool_handler
def smart_read_analyze(
file_path: Annotated[str, Field(description="Path to the file to size and classify before reading it.")],
) -> str:
"""Analyze a file and recommend optimal reading strategy.
Returns file size, line count, and strategy (small/medium/large/very_large)
with specific offset/limit recommendations.
Use this when unsure whether a file is safe to read in full. It only reads
file metadata and line counts, never the content.
"""
path = Path(file_path)
if not path.exists():
return to_json({"success": False, "error": f"File not found: {file_path}"})
size = path.stat().st_size
size_kb = size / 1024
try:
line_count = _count_lines(path)
except OSError as exc:
_log_event(
logging.WARNING,
"line_count_failed",
file_path=file_path,
error_type=type(exc).__name__,
error=str(exc)[:200],
)
line_count = 0
if line_count == 0:
strategy = {"type": "binary", "recommendation": "Binary file - use metadata only"}
elif line_count < 100:
strategy = {
"type": "small",
"recommendation": "Read full content",
"params": {},
}
elif line_count < 500:
strategy = {
"type": "medium",
"recommendation": "Read full or with limit",
"params": {"limit": 500},
}
elif line_count < 2000:
strategy = {
"type": "large",
"recommendation": "Use offset/limit - read in chunks",
"params": {"offset": 0, "limit": 500},
"alternative": "Use ast_navigate_code for code files",
}
else:
strategy = {
"type": "very_large",
"recommendation": "Use Grep for targeted search or AST for structure",
"params": {"offset": 0, "limit": 500},
"alternative": "Grep with head_limit=100 for specific patterns",
}
return to_json({
"success": True,
"file": file_path,
"size_bytes": size,
"size_kb": round(size_kb, 2),
"lines": line_count,
"strategy": strategy,
"estimated_tokens": line_count * 80,
})
# =============================================================================
# TOOL 4 & 5: CONTEXT DEDUPLICATION
# =============================================================================
def _fingerprint(text: str) -> str:
"""Hash a normalized line for duplicate detection.
SHA-256 is used rather than MD5 because MD5 is unavailable when the host
OpenSSL runs in FIPS mode, which would make the whole tool raise.
Args:
text: Normalized line text.
Returns:
Hex digest string.
"""
return hashlib.sha256(text.encode("utf-8", errors="ignore")).hexdigest()
def _load_contexts(contexts: str):
"""Parse and size-check the contexts JSON payload.
Args:
contexts: JSON object string with document text values.
Returns:
Tuple of (parsed dict or None, error string or None).
"""
if len(contexts) > _MAX_CONTEXT_BYTES:
return None, (
"contexts payload is %d characters, exceeding the limit of %d. "
"Split the documents or raise TOKEN_OPTIMIZER_MAX_CONTEXT_BYTES."
% (len(contexts), _MAX_CONTEXT_BYTES)
)
try:
parsed = json.loads(contexts)
except (json.JSONDecodeError, TypeError) as exc:
return None, "Invalid JSON: %s" % exc
if not isinstance(parsed, dict):
return None, "contexts must be a JSON object"
return parsed, None
@_tool(
annotations=_annotations("Deduplicate context documents", True, False, True),
structured_output=False,
)
@mcp_tool_handler
def deduplicate_context(
contexts: Annotated[str, Field(description="JSON object string with any of the keys 'srs', 'readme', 'claude_md', whose values are the full document texts. Keys are processed in that priority order, so a line is kept in the highest-priority document that contains it.")],
) -> str:
"""Remove duplicate content across SRS, README, and CLAUDE.md.
Only applies deduplication if it saves > 20% of total context space.
Uses fingerprint-based line hashing with priority order.
Use this before injecting several overlapping project documents into one
prompt. It returns new text and never writes to disk. Use dedup_estimate
first if you only want to know whether it would help.
"""
ctx, err = _load_contexts(contexts)
if err:
return to_json({"success": False, "error": err})
texts = {}
for key in _DEDUP_PRIORITY:
val = ctx.get(key)
if val and isinstance(val, str) and val.strip():
texts[key] = val
if len(texts) < 2:
return to_json({"success": True, "dedup_applied": False, "reason": "Less than 2 docs"})
original_size = sum(len(t.encode("utf-8", errors="ignore")) for t in texts.values())
if original_size == 0:
return to_json({"success": True, "dedup_applied": False, "reason": "Empty content"})
seen_fps = set()
deduped = {}
removed_bytes = 0
for key in _DEDUP_PRIORITY:
if key not in texts:
continue
lines = texts[key].splitlines(keepends=True)
kept = []
for line in lines:
normalized = line.strip().lower()
if not normalized:
kept.append(line)
continue
fp = _fingerprint(normalized)
if fp in seen_fps:
removed_bytes += len(line.encode("utf-8", errors="ignore"))
else:
seen_fps.add(fp)
kept.append(line)
deduped[key] = "".join(kept)
new_size = original_size - removed_bytes
ratio = removed_bytes / original_size if original_size > 0 else 0.0
if ratio >= _DEDUP_MIN_SAVINGS:
result_ctx = dict(ctx)
for key, text in deduped.items():
result_ctx[key] = text
return to_json({
"success": True,
"dedup_applied": True,
"savings_ratio": round(ratio, 3),
"original_bytes": original_size,
"new_bytes": new_size,
"removed_bytes": removed_bytes,
"deduped_contexts": result_ctx,
})
else:
return to_json({
"success": True,
"dedup_applied": False,
"savings_ratio": round(ratio, 3),
"original_bytes": original_size,
"reason": f"Savings {ratio*100:.1f}% < {_DEDUP_MIN_SAVINGS*100:.0f}% threshold",
})
@_tool(
annotations=_annotations("Estimate deduplication savings", True, False, True),
structured_output=False,
)
@mcp_tool_handler
def dedup_estimate(
contexts: Annotated[str, Field(description="JSON object string with any of the keys 'srs', 'readme', 'claude_md', whose values are the full document texts.")],
) -> str:
"""Estimate deduplication savings without actually deduplicating.
Use this to decide whether calling deduplicate_context is worthwhile. It
returns only the ratio and byte counts, never the rewritten documents.
"""
ctx, err = _load_contexts(contexts)
if err:
return to_json({"success": False, "error": err})
texts = {}
for key in _DEDUP_PRIORITY:
val = ctx.get(key)
if val and isinstance(val, str) and val.strip():
texts[key] = val
if len(texts) < 2:
return to_json({"success": True, "savings_ratio": 0, "original_bytes": 0})
original_size = sum(len(t.encode("utf-8", errors="ignore")) for t in texts.values())
seen_fps = set()
removed = 0
for key in _DEDUP_PRIORITY:
if key not in texts:
continue
for line in texts[key].splitlines(keepends=True):
normalized = line.strip().lower()
if not normalized:
continue
fp = _fingerprint(normalized)
if fp in seen_fps:
removed += len(line.encode("utf-8", errors="ignore"))
else:
seen_fps.add(fp)
ratio = removed / original_size if original_size > 0 else 0.0
return to_json({
"success": True,
"savings_ratio": round(ratio, 3),
"savings_pct": f"{ratio*100:.1f}%",
"original_bytes": original_size,
"estimated_new_bytes": original_size - removed,
"would_apply": ratio >= _DEDUP_MIN_SAVINGS,
})
# =============================================================================
# TOOL 6: CONTEXT BUDGET STATUS
# =============================================================================
@_tool(
annotations=_annotations("Context budget status", True, False, True),
structured_output=False,
)
@mcp_tool_handler
def context_budget_status() -> str:
"""Check current context budget usage (logs + sessions directories).
Monitors the memory directory's size against a 200KB budget and alerts
when usage exceeds 85%.
Use this to decide whether stored session data needs archiving. It is
read-only. The directory walk is capped (default 50000 files) so a very
large tree cannot make this call run unbounded; the response reports
whether the cap was hit.
"""
total_bytes = 0
file_count = 0
truncated = False
if MEMORY_PATH.exists():
for entry in MEMORY_PATH.rglob("*"):
if file_count >= _MAX_BUDGET_SCAN_FILES:
truncated = True
_log_event(
logging.WARNING,
"budget_scan_truncated",
path=str(MEMORY_PATH),
cap=_MAX_BUDGET_SCAN_FILES,
)
break
try:
if entry.is_file():
total_bytes += entry.stat().st_size
file_count += 1
except OSError as exc:
_log_event(
logging.DEBUG,
"budget_scan_stat_skipped",
path=str(entry),
error=str(exc)[:200],
)
usage_pct = (total_bytes / CONTEXT_BUDGET_BYTES) * 100 if CONTEXT_BUDGET_BYTES > 0 else 0
alert = usage_pct >= 85
return to_json({