Upload via GUI (324 Dateien)
This commit is contained in:
@@ -0,0 +1,9 @@
|
||||
"""Caveman compress scripts.
|
||||
|
||||
This package provides tools to compress natural language markdown files
|
||||
into caveman format to save input tokens.
|
||||
"""
|
||||
|
||||
__all__ = ["cli", "compress", "detect", "validate"]
|
||||
|
||||
__version__ = "1.0.0"
|
||||
@@ -0,0 +1,3 @@
|
||||
from .cli import main
|
||||
|
||||
main()
|
||||
@@ -0,0 +1,80 @@
|
||||
#!/usr/bin/env python3
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
# Support both direct execution and module import
|
||||
try:
|
||||
from .validate import validate
|
||||
except ImportError:
|
||||
sys.path.insert(0, str(Path(__file__).parent))
|
||||
from validate import validate
|
||||
|
||||
try:
|
||||
import tiktoken
|
||||
_enc = tiktoken.get_encoding("o200k_base")
|
||||
except ImportError:
|
||||
_enc = None
|
||||
|
||||
|
||||
def count_tokens(text):
|
||||
if _enc is None:
|
||||
return len(text.split()) # fallback: word count
|
||||
return len(_enc.encode(text))
|
||||
|
||||
|
||||
def benchmark_pair(orig_path: Path, comp_path: Path):
|
||||
orig_text = orig_path.read_text(encoding="utf-8", errors="ignore")
|
||||
comp_text = comp_path.read_text(encoding="utf-8", errors="ignore")
|
||||
|
||||
orig_tokens = count_tokens(orig_text)
|
||||
comp_tokens = count_tokens(comp_text)
|
||||
saved = 100 * (orig_tokens - comp_tokens) / orig_tokens if orig_tokens > 0 else 0.0
|
||||
result = validate(orig_path, comp_path)
|
||||
|
||||
return (comp_path.name, orig_tokens, comp_tokens, saved, result.is_valid)
|
||||
|
||||
|
||||
def print_table(rows):
|
||||
print("\n| File | Original | Compressed | Saved % | Valid |")
|
||||
print("|------|----------|------------|---------|-------|")
|
||||
for r in rows:
|
||||
print(f"| {r[0]} | {r[1]} | {r[2]} | {r[3]:.1f}% | {'✅' if r[4] else '❌'} |")
|
||||
|
||||
|
||||
def main():
|
||||
# Direct file pair: python3 benchmark.py original.md compressed.md
|
||||
if len(sys.argv) == 3:
|
||||
orig = Path(sys.argv[1]).resolve()
|
||||
comp = Path(sys.argv[2]).resolve()
|
||||
if not orig.exists():
|
||||
print(f"❌ Not found: {orig}")
|
||||
sys.exit(1)
|
||||
if not comp.exists():
|
||||
print(f"❌ Not found: {comp}")
|
||||
sys.exit(1)
|
||||
print_table([benchmark_pair(orig, comp)])
|
||||
return
|
||||
|
||||
# Glob mode: repo_root/tests/caveman-compress/
|
||||
# __file__ lives at <repo_root>/skills/caveman-compress/scripts/benchmark.py
|
||||
# Walk up four dirs: scripts → caveman-compress → skills → repo_root.
|
||||
tests_dir = Path(__file__).resolve().parents[3] / "tests" / "caveman-compress"
|
||||
if not tests_dir.exists():
|
||||
print(f"❌ Tests dir not found: {tests_dir}")
|
||||
sys.exit(1)
|
||||
|
||||
rows = []
|
||||
for orig in sorted(tests_dir.glob("*.original.md")):
|
||||
comp = orig.with_name(orig.stem.removesuffix(".original") + ".md")
|
||||
if comp.exists():
|
||||
rows.append(benchmark_pair(orig, comp))
|
||||
|
||||
if not rows:
|
||||
print("No compressed file pairs found.")
|
||||
return
|
||||
|
||||
print_table(rows)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,86 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Caveman Compress CLI
|
||||
|
||||
Usage:
|
||||
caveman <filepath>
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
|
||||
# Force UTF-8 on stdout/stderr before any code can print. Windows consoles
|
||||
# default to cp1252 and crash on the ❌ glyphs in error/validation branches,
|
||||
# masking the real error and leaving the user with a half-compressed file.
|
||||
for _stream in (sys.stdout, sys.stderr):
|
||||
reconfigure = getattr(_stream, "reconfigure", None)
|
||||
if callable(reconfigure):
|
||||
try:
|
||||
reconfigure(encoding="utf-8", errors="replace")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
# Direct script execution has no package context. Resolve imports relative to
|
||||
# this installed skill, independent of the caller's working directory.
|
||||
if __package__ in (None, ""):
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
||||
__package__ = "scripts"
|
||||
|
||||
from .compress import backup_dir_for, compress_file
|
||||
from .detect import detect_file_type, should_compress
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Compress a natural-language file with validation and an original backup.")
|
||||
parser.add_argument("filepath", type=Path, help="path to the file to compress")
|
||||
filepath = parser.parse_args().filepath
|
||||
|
||||
# Check file exists
|
||||
if not filepath.exists():
|
||||
print(f"❌ File not found: {filepath}")
|
||||
sys.exit(1)
|
||||
|
||||
if not filepath.is_file():
|
||||
print(f"❌ Not a file: {filepath}")
|
||||
sys.exit(1)
|
||||
|
||||
filepath = filepath.resolve()
|
||||
|
||||
# Detect file type
|
||||
file_type = detect_file_type(filepath)
|
||||
|
||||
print(f"Detected: {file_type}")
|
||||
|
||||
# Check if compressible
|
||||
if not should_compress(filepath):
|
||||
print("Skipping: file is not natural language (code/config)")
|
||||
sys.exit(0)
|
||||
|
||||
print("Starting caveman compression...\n")
|
||||
|
||||
try:
|
||||
success = compress_file(filepath)
|
||||
|
||||
if success:
|
||||
print("\nCompression completed successfully")
|
||||
backup_path = backup_dir_for(filepath) / (filepath.stem + ".original.md")
|
||||
print(f"Compressed: {filepath}")
|
||||
print(f"Original: {backup_path}")
|
||||
sys.exit(0)
|
||||
else:
|
||||
print("\n❌ Compression failed after retries")
|
||||
sys.exit(2)
|
||||
|
||||
except KeyboardInterrupt:
|
||||
print("\nInterrupted by user")
|
||||
sys.exit(130)
|
||||
|
||||
except Exception as e:
|
||||
print(f"\n❌ Error: {e}")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,144 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Detect whether a file is natural language (compressible) or code/config (skip)."""
|
||||
|
||||
import json
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
# Extensions that are natural language and compressible. `.mdc` is a Cursor
|
||||
# rule: prose with YAML frontmatter, the same shape as a SKILL.md, and the
|
||||
# format caveman's own `--with-init` writes to `.cursor/rules/caveman.mdc`.
|
||||
# Content sniffing below runs only for EXTENSIONLESS files, so an extension
|
||||
# that is on neither list falls through to `unknown` and is never compressed.
|
||||
COMPRESSIBLE_EXTENSIONS = {".md", ".mdc", ".txt", ".markdown", ".rst", ".typ", ".typst", ".tex"}
|
||||
|
||||
# Extensions that are code/config and should be skipped
|
||||
SKIP_EXTENSIONS = {
|
||||
".py", ".js", ".ts", ".tsx", ".jsx", ".json", ".yaml", ".yml",
|
||||
".toml", ".env", ".lock", ".css", ".scss", ".html", ".xml",
|
||||
".sql", ".sh", ".bash", ".zsh", ".go", ".rs", ".java", ".c",
|
||||
".cpp", ".h", ".hpp", ".rb", ".php", ".swift", ".kt", ".lua",
|
||||
".dockerfile", ".makefile", ".csv", ".ini", ".cfg",
|
||||
}
|
||||
|
||||
# Well-known build/config files that carry no (or a misleading) extension —
|
||||
# `Dockerfile` has no suffix so `.dockerfile` above never matches it, and
|
||||
# `CMakeLists.txt` would ride the compressible `.txt` rule. Checked by
|
||||
# basename before any extension rule.
|
||||
KNOWN_CODE_FILENAMES = {
|
||||
"dockerfile", "containerfile", "makefile", "gnumakefile", "jenkinsfile",
|
||||
"vagrantfile", "rakefile", "gemfile", "justfile", "procfile", "brewfile",
|
||||
"earthfile", "fastfile", "podfile",
|
||||
"cmakelists.txt",
|
||||
}
|
||||
|
||||
# Patterns that indicate a line is code
|
||||
CODE_PATTERNS = [
|
||||
re.compile(r"^\s*(import |from .+ import |require\(|const |let |var )"),
|
||||
re.compile(r"^\s*(def |class |function |async function |export )"),
|
||||
re.compile(r"^\s*(if\s*\(|for\s*\(|while\s*\(|switch\s*\(|try\s*\{)"),
|
||||
re.compile(r"^\s*[\}\]\);]+\s*$"), # closing braces/brackets
|
||||
re.compile(r"^\s*@\w+"), # decorators/annotations
|
||||
re.compile(r'^\s*"[^"]+"\s*:\s*'), # JSON-like key-value
|
||||
re.compile(r"^\s*\w+\s*=\s*[{\[\(\"']"), # assignment with literal
|
||||
]
|
||||
|
||||
|
||||
def _is_code_line(line: str) -> bool:
|
||||
"""Check if a line looks like code."""
|
||||
return any(p.match(line) for p in CODE_PATTERNS)
|
||||
|
||||
|
||||
def _is_json_content(text: str) -> bool:
|
||||
"""Check if content is valid JSON."""
|
||||
try:
|
||||
json.loads(text)
|
||||
return True
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
return False
|
||||
|
||||
|
||||
def _is_yaml_content(lines: list[str]) -> bool:
|
||||
"""Heuristic: check if content looks like YAML."""
|
||||
yaml_indicators = 0
|
||||
for line in lines[:30]:
|
||||
stripped = line.strip()
|
||||
if stripped.startswith("---"):
|
||||
yaml_indicators += 1
|
||||
elif re.match(r"^\w[\w\s]*:\s", stripped):
|
||||
yaml_indicators += 1
|
||||
elif stripped.startswith("- ") and ":" in stripped:
|
||||
yaml_indicators += 1
|
||||
# If most non-empty lines look like YAML
|
||||
non_empty = sum(1 for l in lines[:30] if l.strip())
|
||||
return non_empty > 0 and yaml_indicators / non_empty > 0.6
|
||||
|
||||
|
||||
def detect_file_type(filepath: Path) -> str:
|
||||
"""Classify a file as 'natural_language', 'code', 'config', or 'unknown'.
|
||||
|
||||
Returns:
|
||||
One of: 'natural_language', 'code', 'config', 'unknown'
|
||||
"""
|
||||
ext = filepath.suffix.lower()
|
||||
|
||||
# Known code filenames win over any extension rule
|
||||
if filepath.name.lower() in KNOWN_CODE_FILENAMES:
|
||||
return "code"
|
||||
|
||||
# Extension-based classification
|
||||
if ext in COMPRESSIBLE_EXTENSIONS:
|
||||
return "natural_language"
|
||||
if ext in SKIP_EXTENSIONS:
|
||||
return "code" if ext not in {".json", ".yaml", ".yml", ".toml", ".ini", ".cfg", ".env"} else "config"
|
||||
|
||||
# Extensionless files (like CLAUDE.md, TODO) — check content
|
||||
if not ext:
|
||||
try:
|
||||
text = filepath.read_text(encoding="utf-8", errors="ignore")
|
||||
except (OSError, PermissionError):
|
||||
return "unknown"
|
||||
|
||||
lines = text.splitlines()[:50]
|
||||
|
||||
# Shebang means executable script, never prose
|
||||
if text.startswith("#!"):
|
||||
return "code"
|
||||
|
||||
if _is_json_content(text[:10000]):
|
||||
return "config"
|
||||
if _is_yaml_content(lines):
|
||||
return "config"
|
||||
|
||||
code_lines = sum(1 for l in lines if l.strip() and _is_code_line(l))
|
||||
non_empty = sum(1 for l in lines if l.strip())
|
||||
if non_empty > 0 and code_lines / non_empty > 0.4:
|
||||
return "code"
|
||||
|
||||
return "natural_language"
|
||||
|
||||
return "unknown"
|
||||
|
||||
|
||||
def should_compress(filepath: Path) -> bool:
|
||||
"""Return True if the file is natural language and should be compressed."""
|
||||
if not filepath.is_file():
|
||||
return False
|
||||
# Skip backup files
|
||||
if filepath.name.endswith(".original.md"):
|
||||
return False
|
||||
return detect_file_type(filepath) == "natural_language"
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import sys
|
||||
|
||||
if len(sys.argv) < 2:
|
||||
print("Usage: python detect.py <file1> [file2] ...")
|
||||
sys.exit(1)
|
||||
|
||||
for path_str in sys.argv[1:]:
|
||||
p = Path(path_str).resolve()
|
||||
file_type = detect_file_type(p)
|
||||
compress = should_compress(p)
|
||||
print(f" {p.name:30s} type={file_type:20s} compress={compress}")
|
||||
@@ -0,0 +1,434 @@
|
||||
#!/usr/bin/env python3
|
||||
import re
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
|
||||
URL_REGEX = re.compile(r"https?://[^\s)`]+") # a backtick is never part of a URL
|
||||
FENCE_OPEN_REGEX = re.compile(r"^(\s{0,3})(`{3,}|~{3,})(.*)$")
|
||||
|
||||
# A line that is nothing but a fence marker plus an optional info string, at ANY
|
||||
# indentation. Used ONLY to scrub leaked markers before inline-code pairing (see
|
||||
# extract_inline_codes) — never for block extraction.
|
||||
#
|
||||
# Widening FENCE_OPEN_REGEX itself to `\s*` looks like the obvious fix for #820
|
||||
# and is a net regression: a lone indented ``` (the natural way to SHOW a fence
|
||||
# inside prose) then opens a block that runs to EOF, swallowing real code blocks
|
||||
# and silently removing their inline spans from validation. That turns a
|
||||
# false-failure bug into a false-PASS bug, and a false PASS overwrites the
|
||||
# user's file with unvalidated output.
|
||||
FENCE_MARKER_LINE_REGEX = re.compile(r"^\s*(?:`{3,}|~{3,})[^`~]*$")
|
||||
|
||||
# Cap on how much of a lost/added span is echoed in an error message. Unpaired
|
||||
# backticks can make a "span" hundreds of characters of prose; printing it whole
|
||||
# is what made #820's failures undiagnosable.
|
||||
MAX_REPORTED_SPAN = 60
|
||||
HEADING_REGEX = re.compile(r"^(#{1,6})\s+(.*)", re.MULTILINE)
|
||||
BULLET_REGEX = re.compile(r"^\s*[-*+]\s+", re.MULTILINE)
|
||||
# Any list item, ordered or not. Four spaces inside a list item is the item's
|
||||
# content indentation, never an indented code block.
|
||||
LIST_ITEM_REGEX = re.compile(r"^\s*(?:[-*+]|\d+[.)])\s")
|
||||
|
||||
# crude but effective path detection
|
||||
# Requires either a path prefix (./ ../ / or drive letter) or a slash/backslash within the match
|
||||
PATH_REGEX = re.compile(r"(?:\./|\.\./|/|[A-Za-z]:\\)[\w\-/\\\.]+|[\w\-\.]+[/\\][\w\-/\\\.]+")
|
||||
|
||||
# PATH_REGEX is crude on purpose and also matches ordinary prose pairs —
|
||||
# "pros/cons", "Node/browser", "state/lifecycle". Caveman prose ADDS those
|
||||
# constructions freely and dropping one breaks nothing, so only an unambiguous
|
||||
# path — a leading ./ ../ / or drive letter, or a dotted filename in the last
|
||||
# component — is treated as a hard loss.
|
||||
DEFINITE_PATH_REGEX = re.compile(r"^(?:\./|\.\./|/|[A-Za-z]:\\)|[^/\\]*\.[A-Za-z0-9]{1,8}$")
|
||||
|
||||
|
||||
class ValidationResult:
|
||||
def __init__(self):
|
||||
self.is_valid = True
|
||||
self.errors = []
|
||||
self.warnings = []
|
||||
|
||||
def add_error(self, msg):
|
||||
self.is_valid = False
|
||||
self.errors.append(msg)
|
||||
|
||||
def add_warning(self, msg):
|
||||
self.warnings.append(msg)
|
||||
|
||||
|
||||
def read_file(path: Path) -> str:
|
||||
return path.read_text(encoding="utf-8")
|
||||
|
||||
|
||||
# ---------- Extractors ----------
|
||||
|
||||
|
||||
def extract_headings(text):
|
||||
return [(level, title.strip()) for level, title in HEADING_REGEX.findall(text)]
|
||||
|
||||
|
||||
def extract_code_blocks(text):
|
||||
"""Line-based fenced code block extractor.
|
||||
|
||||
Handles ``` and ~~~ fences with variable length (CommonMark: closing
|
||||
fence must use same char and be at least as long as opening). Supports
|
||||
nested fences (e.g. an outer 4-backtick block wrapping inner 3-backtick
|
||||
content).
|
||||
|
||||
Merges fenced and indented blocks by their DOCUMENT POSITION, not by
|
||||
type. validate_code_blocks compares this list positionally (c1 != c2),
|
||||
so a type-grouped return (all fenced, then all indented) makes two
|
||||
blocks that swapped RELATIVE order in the document produce the same
|
||||
list on both sides: the comparison passes while the invariant it
|
||||
exists to check (block order preserved) has actually been violated.
|
||||
"""
|
||||
blocks = []
|
||||
lines = text.split("\n")
|
||||
i = 0
|
||||
n = len(lines)
|
||||
while i < n:
|
||||
m = FENCE_OPEN_REGEX.match(lines[i])
|
||||
if not m:
|
||||
i += 1
|
||||
continue
|
||||
start = i
|
||||
fence_char = m.group(2)[0]
|
||||
fence_len = len(m.group(2))
|
||||
open_line = lines[i]
|
||||
block_lines = [open_line]
|
||||
i += 1
|
||||
closed = False
|
||||
while i < n:
|
||||
close_m = FENCE_OPEN_REGEX.match(lines[i])
|
||||
if (
|
||||
close_m
|
||||
and close_m.group(2)[0] == fence_char
|
||||
and len(close_m.group(2)) >= fence_len
|
||||
and close_m.group(3).strip() == ""
|
||||
):
|
||||
block_lines.append(lines[i])
|
||||
closed = True
|
||||
i += 1
|
||||
break
|
||||
block_lines.append(lines[i])
|
||||
i += 1
|
||||
if closed:
|
||||
blocks.append((start, "\n".join(block_lines)))
|
||||
# Unclosed fences are silently skipped — they indicate malformed markdown
|
||||
# and including them would cause false-positive validation failures.
|
||||
ordered = sorted(blocks + extract_indented_code_blocks(text), key=lambda pair: pair[0])
|
||||
return [block_text for _, block_text in ordered]
|
||||
|
||||
|
||||
def extract_indented_code_blocks(text):
|
||||
"""CommonMark indented code blocks — 4-space-indented runs outside any fence.
|
||||
|
||||
Without these, ` kubectl delete pod --all -n prod` was prose to the
|
||||
validator: "code blocks preserved exactly" compared empty to empty and
|
||||
PASSED while the compressor rewrote the command to
|
||||
`kubectl delete pod -n dev`. A clean pass on a mutated destructive command
|
||||
is the worst failure this tool has, because it overwrites the user's file.
|
||||
|
||||
Deliberately conservative about lists: inside a list item, four spaces are
|
||||
the item's content indentation, not code, and nested bullets are ordinary
|
||||
prose the compressor SHOULD rewrite. A run is only treated as code when the
|
||||
document is not inside a list and the run is preceded by a blank line — so
|
||||
this adds detections, it never turns existing passes into false failures.
|
||||
|
||||
Returns (start_line, text) pairs rather than bare text: extract_code_blocks
|
||||
needs each block's document position to merge it with fenced blocks in
|
||||
order (see that function's docstring).
|
||||
"""
|
||||
blocks = []
|
||||
lines = text.split("\n")
|
||||
fenced = set()
|
||||
for block in extract_fenced_spans(lines):
|
||||
fenced.update(block)
|
||||
in_list = False
|
||||
previous_blank = True
|
||||
i = 0
|
||||
n = len(lines)
|
||||
while i < n:
|
||||
line = lines[i]
|
||||
stripped = line.strip()
|
||||
if i in fenced:
|
||||
in_list, previous_blank = in_list, False
|
||||
i += 1
|
||||
continue
|
||||
if not stripped:
|
||||
previous_blank = True
|
||||
i += 1
|
||||
continue
|
||||
indent = len(line) - len(line.lstrip(" \t"))
|
||||
if LIST_ITEM_REGEX.match(line):
|
||||
in_list = True
|
||||
elif indent == 0:
|
||||
in_list = False
|
||||
if not in_list and previous_blank and indent >= 4:
|
||||
start = i
|
||||
run = []
|
||||
while i < n and i not in fenced:
|
||||
current = lines[i]
|
||||
if not current.strip():
|
||||
# A blank line continues an indented block only if more
|
||||
# indented content follows.
|
||||
lookahead = i + 1
|
||||
while lookahead < n and not lines[lookahead].strip():
|
||||
lookahead += 1
|
||||
if lookahead < n and lookahead not in fenced and \
|
||||
len(lines[lookahead]) - len(lines[lookahead].lstrip(" \t")) >= 4:
|
||||
run.extend(lines[i:lookahead])
|
||||
i = lookahead
|
||||
continue
|
||||
break
|
||||
if len(current) - len(current.lstrip(" \t")) < 4:
|
||||
break
|
||||
run.append(current)
|
||||
i += 1
|
||||
if run:
|
||||
blocks.append((start, "\n".join(run)))
|
||||
previous_blank = False
|
||||
continue
|
||||
previous_blank = False
|
||||
i += 1
|
||||
return blocks
|
||||
|
||||
|
||||
def extract_fenced_spans(lines):
|
||||
"""Line-index ranges covered by fenced blocks, so indented-code detection
|
||||
never reaches inside one."""
|
||||
spans = []
|
||||
i = 0
|
||||
n = len(lines)
|
||||
while i < n:
|
||||
m = FENCE_OPEN_REGEX.match(lines[i])
|
||||
if not m:
|
||||
i += 1
|
||||
continue
|
||||
fence_char = m.group(2)[0]
|
||||
fence_len = len(m.group(2))
|
||||
start = i
|
||||
i += 1
|
||||
while i < n:
|
||||
close_m = FENCE_OPEN_REGEX.match(lines[i])
|
||||
if (
|
||||
close_m
|
||||
and close_m.group(2)[0] == fence_char
|
||||
and len(close_m.group(2)) >= fence_len
|
||||
and close_m.group(3).strip() == ""
|
||||
):
|
||||
i += 1
|
||||
break
|
||||
i += 1
|
||||
spans.append(range(start, i))
|
||||
return spans
|
||||
|
||||
|
||||
def extract_urls(text):
|
||||
return set(URL_REGEX.findall(text))
|
||||
|
||||
|
||||
def extract_paths(text):
|
||||
return set(PATH_REGEX.findall(text))
|
||||
|
||||
|
||||
def count_bullets(text):
|
||||
return len(BULLET_REGEX.findall(text))
|
||||
|
||||
|
||||
def extract_inline_codes(text):
|
||||
"""Backtick-delimited inline spans, with fenced code blocks stripped first.
|
||||
|
||||
Previously used a column-0-anchored regex to strip fences, which misses
|
||||
fences indented 1-3 spaces (valid CommonMark). Reuse extract_code_blocks
|
||||
(FENCE_OPEN_REGEX-based, indentation-aware) instead so an indented fence's
|
||||
body backticks don't leak into inline-code pairing.
|
||||
|
||||
Any fence-marker line that survives that pass is then blanked (#820). A
|
||||
fence indented 4+ spaces — what you get from showing an example inside a
|
||||
bullet — is not matched by FENCE_OPEN_REGEX, so extract_code_blocks does
|
||||
not remove it and its OWN backticks used to leak in and shift the pairing
|
||||
of every following span, making the file permanently uncompressible.
|
||||
Blanking just the marker lines fixes that without removing any prose, and
|
||||
cannot run away the way a widened fence opener does.
|
||||
|
||||
The span pattern deliberately still spans newlines. CommonMark permits a
|
||||
line ending inside a code span and hard-wrapped markdown produces them, so
|
||||
a single-line pattern silently drops real spans — which downgrades a
|
||||
deleted or mutated span from error to PASS. Long/garbled spans are a
|
||||
presentation problem, handled by truncating in the error message instead.
|
||||
"""
|
||||
text_without_fences = text
|
||||
for block in extract_code_blocks(text):
|
||||
text_without_fences = text_without_fences.replace(block, "", 1)
|
||||
text_without_fences = "\n".join(
|
||||
"" if FENCE_MARKER_LINE_REGEX.match(line) else line
|
||||
for line in text_without_fences.split("\n")
|
||||
)
|
||||
# Run-aware pairing (CommonMark): a span opens with a backtick run and
|
||||
# closes with an equal-length run. A mid-line ``` in prose ("inside a
|
||||
# ```diff fence") has no equal closer, so it matches nothing — instead of
|
||||
# its trailing backtick shifting the pairing of every following span.
|
||||
# The opener is the WHOLE run (`(?!`)` after it), so a lone ``` can never
|
||||
# shrink to a single backtick and pair with the next span. The content may
|
||||
# hold a run of a different length (``a`b``), so it is "anything, lazily"
|
||||
# up to the first equal run, not "anything but a backtick"; DOTALL keeps
|
||||
# hard-wrapped spans in the comparison set.
|
||||
return [
|
||||
m.group(2)
|
||||
for m in re.finditer(r"(?<!`)(`+)(?!`)(.+?)(?<!`)\1(?!`)", text_without_fences, re.DOTALL)
|
||||
]
|
||||
|
||||
|
||||
# ---------- Validators ----------
|
||||
|
||||
|
||||
def validate_headings(orig, comp, result):
|
||||
h1 = extract_headings(orig)
|
||||
h2 = extract_headings(comp)
|
||||
|
||||
# Changed heading TEXT is an error, not a warning. Every in-document anchor
|
||||
# link points at a heading's slug, so renaming "# Configuration Options" to
|
||||
# "# Config" silently breaks all of them — and SKILL.md and CLAUDE.md both
|
||||
# state headings are preserved. Only counts used to gate the overwrite, so
|
||||
# a run that renamed every heading reported "Validation passed". A level-only
|
||||
# change keeps every slug intact and stays a warning.
|
||||
if len(h1) != len(h2):
|
||||
result.add_error(f"Heading count mismatch: {len(h1)} vs {len(h2)}")
|
||||
return
|
||||
|
||||
t1 = [text for _, text in h1]
|
||||
t2 = [text for _, text in h2]
|
||||
if t1 != t2:
|
||||
lost = [t for t in t1 if t not in t2]
|
||||
added = [t for t in t2 if t not in t1]
|
||||
result.add_error(f"Heading text/order changed: lost={lost}, added={added}")
|
||||
elif h1 != h2:
|
||||
# Same text, different level. The outline moved but no anchor broke —
|
||||
# slugs come from the text, so links still resolve.
|
||||
result.add_warning("Heading levels changed")
|
||||
|
||||
|
||||
def validate_code_blocks(orig, comp, result):
|
||||
c1 = extract_code_blocks(orig)
|
||||
c2 = extract_code_blocks(comp)
|
||||
|
||||
if c1 != c2:
|
||||
result.add_error("Code blocks not preserved exactly")
|
||||
|
||||
|
||||
def validate_urls(orig, comp, result):
|
||||
u1 = extract_urls(orig)
|
||||
u2 = extract_urls(comp)
|
||||
|
||||
if u1 != u2:
|
||||
result.add_error(f"URL mismatch: lost={u1 - u2}, added={u2 - u1}")
|
||||
|
||||
|
||||
def validate_paths(orig, comp, result):
|
||||
"""File paths are preserved — an error, never a warning.
|
||||
|
||||
This validator never called add_error at all, so a compressed file that had
|
||||
dropped `src/hooks/caveman-config.js` still reported "Validation passed" and
|
||||
the in-place overwrite stood. SKILL.md and CLAUDE.md both promise paths
|
||||
survive compression; nothing enforced it.
|
||||
"""
|
||||
p1 = extract_paths(orig)
|
||||
p2 = extract_paths(comp)
|
||||
lost = p1 - p2
|
||||
added = p2 - p1
|
||||
|
||||
definite = {p for p in lost if DEFINITE_PATH_REGEX.search(p)}
|
||||
if definite:
|
||||
result.add_error(f"File paths lost: {sorted(definite)}")
|
||||
if (lost - definite) or added:
|
||||
result.add_warning(f"Path mismatch: lost={sorted(lost)}, added={sorted(added)}")
|
||||
|
||||
|
||||
def validate_bullets(orig, comp, result):
|
||||
b1 = count_bullets(orig)
|
||||
b2 = count_bullets(comp)
|
||||
|
||||
if b1 == 0:
|
||||
return
|
||||
|
||||
diff = abs(b1 - b2) / b1
|
||||
|
||||
if diff > 0.15:
|
||||
result.add_warning(f"Bullet count changed too much: {b1} -> {b2}")
|
||||
|
||||
|
||||
def validate_inline_codes(orig, comp, result):
|
||||
def _render_spans(spans):
|
||||
"""Render spans for an error message, truncated and newline-escaped.
|
||||
|
||||
A span may legitimately contain newlines, and an unpaired backtick can
|
||||
make one hundreds of characters of prose. Printing those whole is what
|
||||
made #820's failures undiagnosable — but the fix belongs here, in
|
||||
presentation, not in what counts as a span.
|
||||
"""
|
||||
out = []
|
||||
for span in sorted(spans):
|
||||
flat = span.replace("\n", "\\n")
|
||||
if len(flat) > MAX_REPORTED_SPAN:
|
||||
flat = flat[:MAX_REPORTED_SPAN] + "…"
|
||||
out.append(repr(flat))
|
||||
return "{" + ", ".join(out) + "}"
|
||||
|
||||
c1 = Counter(extract_inline_codes(orig))
|
||||
c2 = Counter(extract_inline_codes(comp))
|
||||
|
||||
if c1 != c2:
|
||||
lost = set(c1.keys()) - set(c2.keys())
|
||||
added = set(c2.keys()) - set(c1.keys())
|
||||
for code, count in c1.items():
|
||||
if code in c2 and c2[code] < count:
|
||||
lost.add(f"{code} (lost {count - c2[code]} of {count} occurrences)")
|
||||
if lost:
|
||||
result.add_error(f"Inline code lost: {_render_spans(lost)}")
|
||||
if added:
|
||||
result.add_warning(f"Inline code added: {_render_spans(added)}")
|
||||
|
||||
|
||||
# ---------- Main ----------
|
||||
|
||||
|
||||
def validate(original_path: Path, compressed_path: Path) -> ValidationResult:
|
||||
result = ValidationResult()
|
||||
|
||||
orig = read_file(original_path)
|
||||
comp = read_file(compressed_path)
|
||||
|
||||
validate_headings(orig, comp, result)
|
||||
validate_code_blocks(orig, comp, result)
|
||||
validate_urls(orig, comp, result)
|
||||
validate_paths(orig, comp, result)
|
||||
validate_bullets(orig, comp, result)
|
||||
validate_inline_codes(orig, comp, result)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
# ---------- CLI ----------
|
||||
|
||||
if __name__ == "__main__":
|
||||
import sys
|
||||
|
||||
if len(sys.argv) != 3:
|
||||
print("Usage: python validate.py <original> <compressed>")
|
||||
sys.exit(1)
|
||||
|
||||
orig = Path(sys.argv[1]).resolve()
|
||||
comp = Path(sys.argv[2]).resolve()
|
||||
|
||||
res = validate(orig, comp)
|
||||
|
||||
print(f"\nValid: {res.is_valid}")
|
||||
|
||||
if res.errors:
|
||||
print("\nErrors:")
|
||||
for e in res.errors:
|
||||
print(f" - {e}")
|
||||
|
||||
if res.warnings:
|
||||
print("\nWarnings:")
|
||||
for w in res.warnings:
|
||||
print(f" - {w}")
|
||||
Reference in New Issue
Block a user