Upload via GUI (324 Dateien)

This commit is contained in:
2026-10-30 05:54:11 +00:00
parent 4ef589ebbc
commit a70916fc7d
50 changed files with 4159 additions and 0 deletions
@@ -0,0 +1,9 @@
"""Caveman compress scripts.
This package provides tools to compress natural language markdown files
into caveman format to save input tokens.
"""
__all__ = ["cli", "compress", "detect", "validate"]
__version__ = "1.0.0"
@@ -0,0 +1,3 @@
from .cli import main
main()
@@ -0,0 +1,80 @@
#!/usr/bin/env python3
from pathlib import Path
import sys
# Support both direct execution and module import
try:
from .validate import validate
except ImportError:
sys.path.insert(0, str(Path(__file__).parent))
from validate import validate
try:
import tiktoken
_enc = tiktoken.get_encoding("o200k_base")
except ImportError:
_enc = None
def count_tokens(text):
if _enc is None:
return len(text.split()) # fallback: word count
return len(_enc.encode(text))
def benchmark_pair(orig_path: Path, comp_path: Path):
orig_text = orig_path.read_text(encoding="utf-8", errors="ignore")
comp_text = comp_path.read_text(encoding="utf-8", errors="ignore")
orig_tokens = count_tokens(orig_text)
comp_tokens = count_tokens(comp_text)
saved = 100 * (orig_tokens - comp_tokens) / orig_tokens if orig_tokens > 0 else 0.0
result = validate(orig_path, comp_path)
return (comp_path.name, orig_tokens, comp_tokens, saved, result.is_valid)
def print_table(rows):
print("\n| File | Original | Compressed | Saved % | Valid |")
print("|------|----------|------------|---------|-------|")
for r in rows:
print(f"| {r[0]} | {r[1]} | {r[2]} | {r[3]:.1f}% | {'✅' if r[4] else '❌'} |")
def main():
# Direct file pair: python3 benchmark.py original.md compressed.md
if len(sys.argv) == 3:
orig = Path(sys.argv[1]).resolve()
comp = Path(sys.argv[2]).resolve()
if not orig.exists():
print(f"❌ Not found: {orig}")
sys.exit(1)
if not comp.exists():
print(f"❌ Not found: {comp}")
sys.exit(1)
print_table([benchmark_pair(orig, comp)])
return
# Glob mode: repo_root/tests/caveman-compress/
# __file__ lives at <repo_root>/skills/caveman-compress/scripts/benchmark.py
# Walk up four dirs: scripts → caveman-compress → skills → repo_root.
tests_dir = Path(__file__).resolve().parents[3] / "tests" / "caveman-compress"
if not tests_dir.exists():
print(f"❌ Tests dir not found: {tests_dir}")
sys.exit(1)
rows = []
for orig in sorted(tests_dir.glob("*.original.md")):
comp = orig.with_name(orig.stem.removesuffix(".original") + ".md")
if comp.exists():
rows.append(benchmark_pair(orig, comp))
if not rows:
print("No compressed file pairs found.")
return
print_table(rows)
if __name__ == "__main__":
main()
@@ -0,0 +1,86 @@
#!/usr/bin/env python3
"""
Caveman Compress CLI
Usage:
caveman <filepath>
"""
import argparse
import sys
# Force UTF-8 on stdout/stderr before any code can print. Windows consoles
# default to cp1252 and crash on the ❌ glyphs in error/validation branches,
# masking the real error and leaving the user with a half-compressed file.
for _stream in (sys.stdout, sys.stderr):
reconfigure = getattr(_stream, "reconfigure", None)
if callable(reconfigure):
try:
reconfigure(encoding="utf-8", errors="replace")
except Exception:
pass
from pathlib import Path
# Direct script execution has no package context. Resolve imports relative to
# this installed skill, independent of the caller's working directory.
if __package__ in (None, ""):
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
__package__ = "scripts"
from .compress import backup_dir_for, compress_file
from .detect import detect_file_type, should_compress
def main():
parser = argparse.ArgumentParser(description="Compress a natural-language file with validation and an original backup.")
parser.add_argument("filepath", type=Path, help="path to the file to compress")
filepath = parser.parse_args().filepath
# Check file exists
if not filepath.exists():
print(f"❌ File not found: {filepath}")
sys.exit(1)
if not filepath.is_file():
print(f"❌ Not a file: {filepath}")
sys.exit(1)
filepath = filepath.resolve()
# Detect file type
file_type = detect_file_type(filepath)
print(f"Detected: {file_type}")
# Check if compressible
if not should_compress(filepath):
print("Skipping: file is not natural language (code/config)")
sys.exit(0)
print("Starting caveman compression...\n")
try:
success = compress_file(filepath)
if success:
print("\nCompression completed successfully")
backup_path = backup_dir_for(filepath) / (filepath.stem + ".original.md")
print(f"Compressed: {filepath}")
print(f"Original: {backup_path}")
sys.exit(0)
else:
print("\n❌ Compression failed after retries")
sys.exit(2)
except KeyboardInterrupt:
print("\nInterrupted by user")
sys.exit(130)
except Exception as e:
print(f"\n❌ Error: {e}")
sys.exit(1)
if __name__ == "__main__":
main()
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,144 @@
#!/usr/bin/env python3
"""Detect whether a file is natural language (compressible) or code/config (skip)."""
import json
import re
from pathlib import Path
# Extensions that are natural language and compressible. `.mdc` is a Cursor
# rule: prose with YAML frontmatter, the same shape as a SKILL.md, and the
# format caveman's own `--with-init` writes to `.cursor/rules/caveman.mdc`.
# Content sniffing below runs only for EXTENSIONLESS files, so an extension
# that is on neither list falls through to `unknown` and is never compressed.
COMPRESSIBLE_EXTENSIONS = {".md", ".mdc", ".txt", ".markdown", ".rst", ".typ", ".typst", ".tex"}
# Extensions that are code/config and should be skipped
SKIP_EXTENSIONS = {
".py", ".js", ".ts", ".tsx", ".jsx", ".json", ".yaml", ".yml",
".toml", ".env", ".lock", ".css", ".scss", ".html", ".xml",
".sql", ".sh", ".bash", ".zsh", ".go", ".rs", ".java", ".c",
".cpp", ".h", ".hpp", ".rb", ".php", ".swift", ".kt", ".lua",
".dockerfile", ".makefile", ".csv", ".ini", ".cfg",
}
# Well-known build/config files that carry no (or a misleading) extension —
# `Dockerfile` has no suffix so `.dockerfile` above never matches it, and
# `CMakeLists.txt` would ride the compressible `.txt` rule. Checked by
# basename before any extension rule.
KNOWN_CODE_FILENAMES = {
"dockerfile", "containerfile", "makefile", "gnumakefile", "jenkinsfile",
"vagrantfile", "rakefile", "gemfile", "justfile", "procfile", "brewfile",
"earthfile", "fastfile", "podfile",
"cmakelists.txt",
}
# Patterns that indicate a line is code
CODE_PATTERNS = [
re.compile(r"^\s*(import |from .+ import |require\(|const |let |var )"),
re.compile(r"^\s*(def |class |function |async function |export )"),
re.compile(r"^\s*(if\s*\(|for\s*\(|while\s*\(|switch\s*\(|try\s*\{)"),
re.compile(r"^\s*[\}\]\);]+\s*$"), # closing braces/brackets
re.compile(r"^\s*@\w+"), # decorators/annotations
re.compile(r'^\s*"[^"]+"\s*:\s*'), # JSON-like key-value
re.compile(r"^\s*\w+\s*=\s*[{\[\(\"']"), # assignment with literal
]
def _is_code_line(line: str) -> bool:
"""Check if a line looks like code."""
return any(p.match(line) for p in CODE_PATTERNS)
def _is_json_content(text: str) -> bool:
"""Check if content is valid JSON."""
try:
json.loads(text)
return True
except (json.JSONDecodeError, ValueError):
return False
def _is_yaml_content(lines: list[str]) -> bool:
"""Heuristic: check if content looks like YAML."""
yaml_indicators = 0
for line in lines[:30]:
stripped = line.strip()
if stripped.startswith("---"):
yaml_indicators += 1
elif re.match(r"^\w[\w\s]*:\s", stripped):
yaml_indicators += 1
elif stripped.startswith("- ") and ":" in stripped:
yaml_indicators += 1
# If most non-empty lines look like YAML
non_empty = sum(1 for l in lines[:30] if l.strip())
return non_empty > 0 and yaml_indicators / non_empty > 0.6
def detect_file_type(filepath: Path) -> str:
"""Classify a file as 'natural_language', 'code', 'config', or 'unknown'.
Returns:
One of: 'natural_language', 'code', 'config', 'unknown'
"""
ext = filepath.suffix.lower()
# Known code filenames win over any extension rule
if filepath.name.lower() in KNOWN_CODE_FILENAMES:
return "code"
# Extension-based classification
if ext in COMPRESSIBLE_EXTENSIONS:
return "natural_language"
if ext in SKIP_EXTENSIONS:
return "code" if ext not in {".json", ".yaml", ".yml", ".toml", ".ini", ".cfg", ".env"} else "config"
# Extensionless files (like CLAUDE.md, TODO) — check content
if not ext:
try:
text = filepath.read_text(encoding="utf-8", errors="ignore")
except (OSError, PermissionError):
return "unknown"
lines = text.splitlines()[:50]
# Shebang means executable script, never prose
if text.startswith("#!"):
return "code"
if _is_json_content(text[:10000]):
return "config"
if _is_yaml_content(lines):
return "config"
code_lines = sum(1 for l in lines if l.strip() and _is_code_line(l))
non_empty = sum(1 for l in lines if l.strip())
if non_empty > 0 and code_lines / non_empty > 0.4:
return "code"
return "natural_language"
return "unknown"
def should_compress(filepath: Path) -> bool:
"""Return True if the file is natural language and should be compressed."""
if not filepath.is_file():
return False
# Skip backup files
if filepath.name.endswith(".original.md"):
return False
return detect_file_type(filepath) == "natural_language"
if __name__ == "__main__":
import sys
if len(sys.argv) < 2:
print("Usage: python detect.py <file1> [file2] ...")
sys.exit(1)
for path_str in sys.argv[1:]:
p = Path(path_str).resolve()
file_type = detect_file_type(p)
compress = should_compress(p)
print(f" {p.name:30s} type={file_type:20s} compress={compress}")
@@ -0,0 +1,434 @@
#!/usr/bin/env python3
import re
from collections import Counter
from pathlib import Path
URL_REGEX = re.compile(r"https?://[^\s)`]+") # a backtick is never part of a URL
FENCE_OPEN_REGEX = re.compile(r"^(\s{0,3})(`{3,}|~{3,})(.*)$")
# A line that is nothing but a fence marker plus an optional info string, at ANY
# indentation. Used ONLY to scrub leaked markers before inline-code pairing (see
# extract_inline_codes) — never for block extraction.
#
# Widening FENCE_OPEN_REGEX itself to `\s*` looks like the obvious fix for #820
# and is a net regression: a lone indented ``` (the natural way to SHOW a fence
# inside prose) then opens a block that runs to EOF, swallowing real code blocks
# and silently removing their inline spans from validation. That turns a
# false-failure bug into a false-PASS bug, and a false PASS overwrites the
# user's file with unvalidated output.
FENCE_MARKER_LINE_REGEX = re.compile(r"^\s*(?:`{3,}|~{3,})[^`~]*$")
# Cap on how much of a lost/added span is echoed in an error message. Unpaired
# backticks can make a "span" hundreds of characters of prose; printing it whole
# is what made #820's failures undiagnosable.
MAX_REPORTED_SPAN = 60
HEADING_REGEX = re.compile(r"^(#{1,6})\s+(.*)", re.MULTILINE)
BULLET_REGEX = re.compile(r"^\s*[-*+]\s+", re.MULTILINE)
# Any list item, ordered or not. Four spaces inside a list item is the item's
# content indentation, never an indented code block.
LIST_ITEM_REGEX = re.compile(r"^\s*(?:[-*+]|\d+[.)])\s")
# crude but effective path detection
# Requires either a path prefix (./ ../ / or drive letter) or a slash/backslash within the match
PATH_REGEX = re.compile(r"(?:\./|\.\./|/|[A-Za-z]:\\)[\w\-/\\\.]+|[\w\-\.]+[/\\][\w\-/\\\.]+")
# PATH_REGEX is crude on purpose and also matches ordinary prose pairs —
# "pros/cons", "Node/browser", "state/lifecycle". Caveman prose ADDS those
# constructions freely and dropping one breaks nothing, so only an unambiguous
# path — a leading ./ ../ / or drive letter, or a dotted filename in the last
# component — is treated as a hard loss.
DEFINITE_PATH_REGEX = re.compile(r"^(?:\./|\.\./|/|[A-Za-z]:\\)|[^/\\]*\.[A-Za-z0-9]{1,8}$")
class ValidationResult:
def __init__(self):
self.is_valid = True
self.errors = []
self.warnings = []
def add_error(self, msg):
self.is_valid = False
self.errors.append(msg)
def add_warning(self, msg):
self.warnings.append(msg)
def read_file(path: Path) -> str:
return path.read_text(encoding="utf-8")
# ---------- Extractors ----------
def extract_headings(text):
return [(level, title.strip()) for level, title in HEADING_REGEX.findall(text)]
def extract_code_blocks(text):
"""Line-based fenced code block extractor.
Handles ``` and ~~~ fences with variable length (CommonMark: closing
fence must use same char and be at least as long as opening). Supports
nested fences (e.g. an outer 4-backtick block wrapping inner 3-backtick
content).
Merges fenced and indented blocks by their DOCUMENT POSITION, not by
type. validate_code_blocks compares this list positionally (c1 != c2),
so a type-grouped return (all fenced, then all indented) makes two
blocks that swapped RELATIVE order in the document produce the same
list on both sides: the comparison passes while the invariant it
exists to check (block order preserved) has actually been violated.
"""
blocks = []
lines = text.split("\n")
i = 0
n = len(lines)
while i < n:
m = FENCE_OPEN_REGEX.match(lines[i])
if not m:
i += 1
continue
start = i
fence_char = m.group(2)[0]
fence_len = len(m.group(2))
open_line = lines[i]
block_lines = [open_line]
i += 1
closed = False
while i < n:
close_m = FENCE_OPEN_REGEX.match(lines[i])
if (
close_m
and close_m.group(2)[0] == fence_char
and len(close_m.group(2)) >= fence_len
and close_m.group(3).strip() == ""
):
block_lines.append(lines[i])
closed = True
i += 1
break
block_lines.append(lines[i])
i += 1
if closed:
blocks.append((start, "\n".join(block_lines)))
# Unclosed fences are silently skipped — they indicate malformed markdown
# and including them would cause false-positive validation failures.
ordered = sorted(blocks + extract_indented_code_blocks(text), key=lambda pair: pair[0])
return [block_text for _, block_text in ordered]
def extract_indented_code_blocks(text):
"""CommonMark indented code blocks — 4-space-indented runs outside any fence.
Without these, ` kubectl delete pod --all -n prod` was prose to the
validator: "code blocks preserved exactly" compared empty to empty and
PASSED while the compressor rewrote the command to
`kubectl delete pod -n dev`. A clean pass on a mutated destructive command
is the worst failure this tool has, because it overwrites the user's file.
Deliberately conservative about lists: inside a list item, four spaces are
the item's content indentation, not code, and nested bullets are ordinary
prose the compressor SHOULD rewrite. A run is only treated as code when the
document is not inside a list and the run is preceded by a blank line — so
this adds detections, it never turns existing passes into false failures.
Returns (start_line, text) pairs rather than bare text: extract_code_blocks
needs each block's document position to merge it with fenced blocks in
order (see that function's docstring).
"""
blocks = []
lines = text.split("\n")
fenced = set()
for block in extract_fenced_spans(lines):
fenced.update(block)
in_list = False
previous_blank = True
i = 0
n = len(lines)
while i < n:
line = lines[i]
stripped = line.strip()
if i in fenced:
in_list, previous_blank = in_list, False
i += 1
continue
if not stripped:
previous_blank = True
i += 1
continue
indent = len(line) - len(line.lstrip(" \t"))
if LIST_ITEM_REGEX.match(line):
in_list = True
elif indent == 0:
in_list = False
if not in_list and previous_blank and indent >= 4:
start = i
run = []
while i < n and i not in fenced:
current = lines[i]
if not current.strip():
# A blank line continues an indented block only if more
# indented content follows.
lookahead = i + 1
while lookahead < n and not lines[lookahead].strip():
lookahead += 1
if lookahead < n and lookahead not in fenced and \
len(lines[lookahead]) - len(lines[lookahead].lstrip(" \t")) >= 4:
run.extend(lines[i:lookahead])
i = lookahead
continue
break
if len(current) - len(current.lstrip(" \t")) < 4:
break
run.append(current)
i += 1
if run:
blocks.append((start, "\n".join(run)))
previous_blank = False
continue
previous_blank = False
i += 1
return blocks
def extract_fenced_spans(lines):
"""Line-index ranges covered by fenced blocks, so indented-code detection
never reaches inside one."""
spans = []
i = 0
n = len(lines)
while i < n:
m = FENCE_OPEN_REGEX.match(lines[i])
if not m:
i += 1
continue
fence_char = m.group(2)[0]
fence_len = len(m.group(2))
start = i
i += 1
while i < n:
close_m = FENCE_OPEN_REGEX.match(lines[i])
if (
close_m
and close_m.group(2)[0] == fence_char
and len(close_m.group(2)) >= fence_len
and close_m.group(3).strip() == ""
):
i += 1
break
i += 1
spans.append(range(start, i))
return spans
def extract_urls(text):
return set(URL_REGEX.findall(text))
def extract_paths(text):
return set(PATH_REGEX.findall(text))
def count_bullets(text):
return len(BULLET_REGEX.findall(text))
def extract_inline_codes(text):
"""Backtick-delimited inline spans, with fenced code blocks stripped first.
Previously used a column-0-anchored regex to strip fences, which misses
fences indented 1-3 spaces (valid CommonMark). Reuse extract_code_blocks
(FENCE_OPEN_REGEX-based, indentation-aware) instead so an indented fence's
body backticks don't leak into inline-code pairing.
Any fence-marker line that survives that pass is then blanked (#820). A
fence indented 4+ spaces — what you get from showing an example inside a
bullet — is not matched by FENCE_OPEN_REGEX, so extract_code_blocks does
not remove it and its OWN backticks used to leak in and shift the pairing
of every following span, making the file permanently uncompressible.
Blanking just the marker lines fixes that without removing any prose, and
cannot run away the way a widened fence opener does.
The span pattern deliberately still spans newlines. CommonMark permits a
line ending inside a code span and hard-wrapped markdown produces them, so
a single-line pattern silently drops real spans — which downgrades a
deleted or mutated span from error to PASS. Long/garbled spans are a
presentation problem, handled by truncating in the error message instead.
"""
text_without_fences = text
for block in extract_code_blocks(text):
text_without_fences = text_without_fences.replace(block, "", 1)
text_without_fences = "\n".join(
"" if FENCE_MARKER_LINE_REGEX.match(line) else line
for line in text_without_fences.split("\n")
)
# Run-aware pairing (CommonMark): a span opens with a backtick run and
# closes with an equal-length run. A mid-line ``` in prose ("inside a
# ```diff fence") has no equal closer, so it matches nothing — instead of
# its trailing backtick shifting the pairing of every following span.
# The opener is the WHOLE run (`(?!`)` after it), so a lone ``` can never
# shrink to a single backtick and pair with the next span. The content may
# hold a run of a different length (``a`b``), so it is "anything, lazily"
# up to the first equal run, not "anything but a backtick"; DOTALL keeps
# hard-wrapped spans in the comparison set.
return [
m.group(2)
for m in re.finditer(r"(?<!`)(`+)(?!`)(.+?)(?<!`)\1(?!`)", text_without_fences, re.DOTALL)
]
# ---------- Validators ----------
def validate_headings(orig, comp, result):
h1 = extract_headings(orig)
h2 = extract_headings(comp)
# Changed heading TEXT is an error, not a warning. Every in-document anchor
# link points at a heading's slug, so renaming "# Configuration Options" to
# "# Config" silently breaks all of them — and SKILL.md and CLAUDE.md both
# state headings are preserved. Only counts used to gate the overwrite, so
# a run that renamed every heading reported "Validation passed". A level-only
# change keeps every slug intact and stays a warning.
if len(h1) != len(h2):
result.add_error(f"Heading count mismatch: {len(h1)} vs {len(h2)}")
return
t1 = [text for _, text in h1]
t2 = [text for _, text in h2]
if t1 != t2:
lost = [t for t in t1 if t not in t2]
added = [t for t in t2 if t not in t1]
result.add_error(f"Heading text/order changed: lost={lost}, added={added}")
elif h1 != h2:
# Same text, different level. The outline moved but no anchor broke —
# slugs come from the text, so links still resolve.
result.add_warning("Heading levels changed")
def validate_code_blocks(orig, comp, result):
c1 = extract_code_blocks(orig)
c2 = extract_code_blocks(comp)
if c1 != c2:
result.add_error("Code blocks not preserved exactly")
def validate_urls(orig, comp, result):
u1 = extract_urls(orig)
u2 = extract_urls(comp)
if u1 != u2:
result.add_error(f"URL mismatch: lost={u1 - u2}, added={u2 - u1}")
def validate_paths(orig, comp, result):
"""File paths are preserved — an error, never a warning.
This validator never called add_error at all, so a compressed file that had
dropped `src/hooks/caveman-config.js` still reported "Validation passed" and
the in-place overwrite stood. SKILL.md and CLAUDE.md both promise paths
survive compression; nothing enforced it.
"""
p1 = extract_paths(orig)
p2 = extract_paths(comp)
lost = p1 - p2
added = p2 - p1
definite = {p for p in lost if DEFINITE_PATH_REGEX.search(p)}
if definite:
result.add_error(f"File paths lost: {sorted(definite)}")
if (lost - definite) or added:
result.add_warning(f"Path mismatch: lost={sorted(lost)}, added={sorted(added)}")
def validate_bullets(orig, comp, result):
b1 = count_bullets(orig)
b2 = count_bullets(comp)
if b1 == 0:
return
diff = abs(b1 - b2) / b1
if diff > 0.15:
result.add_warning(f"Bullet count changed too much: {b1} -> {b2}")
def validate_inline_codes(orig, comp, result):
def _render_spans(spans):
"""Render spans for an error message, truncated and newline-escaped.
A span may legitimately contain newlines, and an unpaired backtick can
make one hundreds of characters of prose. Printing those whole is what
made #820's failures undiagnosable — but the fix belongs here, in
presentation, not in what counts as a span.
"""
out = []
for span in sorted(spans):
flat = span.replace("\n", "\\n")
if len(flat) > MAX_REPORTED_SPAN:
flat = flat[:MAX_REPORTED_SPAN] + "…"
out.append(repr(flat))
return "{" + ", ".join(out) + "}"
c1 = Counter(extract_inline_codes(orig))
c2 = Counter(extract_inline_codes(comp))
if c1 != c2:
lost = set(c1.keys()) - set(c2.keys())
added = set(c2.keys()) - set(c1.keys())
for code, count in c1.items():
if code in c2 and c2[code] < count:
lost.add(f"{code} (lost {count - c2[code]} of {count} occurrences)")
if lost:
result.add_error(f"Inline code lost: {_render_spans(lost)}")
if added:
result.add_warning(f"Inline code added: {_render_spans(added)}")
# ---------- Main ----------
def validate(original_path: Path, compressed_path: Path) -> ValidationResult:
result = ValidationResult()
orig = read_file(original_path)
comp = read_file(compressed_path)
validate_headings(orig, comp, result)
validate_code_blocks(orig, comp, result)
validate_urls(orig, comp, result)
validate_paths(orig, comp, result)
validate_bullets(orig, comp, result)
validate_inline_codes(orig, comp, result)
return result
# ---------- CLI ----------
if __name__ == "__main__":
import sys
if len(sys.argv) != 3:
print("Usage: python validate.py <original> <compressed>")
sys.exit(1)
orig = Path(sys.argv[1]).resolve()
comp = Path(sys.argv[2]).resolve()
res = validate(orig, comp)
print(f"\nValid: {res.is_valid}")
if res.errors:
print("\nErrors:")
for e in res.errors:
print(f" - {e}")
if res.warnings:
print("\nWarnings:")
for w in res.warnings:
print(f" - {w}")