The cure for AI slop is a 1986 aircraft manual/kit

asd-ste100/tests/test_ste_pregate.py

4 KB·Raw on GitHub ↗

import importlib.util
import json
import subprocess
import sys
import unittest
from pathlib import Path
 
 
ROOT = Path(__file__).resolve().parents[1]
HOOK = ROOT / "hooks" / "ste-pregate.py"
SPEC = importlib.util.spec_from_file_location("ste_pregate", HOOK)
GATE = importlib.util.module_from_spec(SPEC)
SPEC.loader.exec_module(GATE)
 
SLOP = ("Great question! We have leveraged a robust, cutting-edge solution to seamlessly "
        "facilitate the deployment process; furthermore, the file is read by the parser "
        "prior to initialization; it is important to note that this is powerful.")
 
 
def run_hook(command):
    payload = {"tool_name": "Bash", "tool_input": {"command": command}}
    proc = subprocess.run([sys.executable, str(HOOK)], input=json.dumps(payload),
                          capture_output=True, text=True, timeout=30)
    return proc.stdout.strip()
 
 
class CommitDetectionTests(unittest.TestCase):
    def test_message_flag_is_gated(self):
        for command in (
            'git commit -m "Fix the bug"',
            "git commit -m 'Fix the bug'",
            'cd repo && git add . && git commit -m "Fix the bug"',
            'git -c user.name=a commit -m "Fix the bug"',
            'GIT_AUTHOR_NAME=a git commit --message="Fix the bug"',
        ):
            with self.subTest(command=command):
                self.assertEqual(GATE.commit_message(command), "Fix the bug")
 
    def test_commit_boundaries_stay_exact(self):
        cases = (
            ('git commit -m "One" & git commit -m "Two"', "One\n\nTwo"),
            ('git commit -m "One" && git commit -m "Two"', "One\n\nTwo"),
            ('{ git commit -m "Fix the bug"; }', "Fix the bug"),
            ("git commit -F - <<EOF\nFix the bug.\nEOF; echo done", "Fix the bug."),
            ('git commit -m "Fix the bug" 2>&1', "Fix the bug"),
        )
        for command, message in cases:
            with self.subTest(command=command):
                self.assertEqual(GATE.commit_message(command), message)
 
    def test_heredoc_of_the_commit_is_gated(self):
        command = 'git commit -m "$(cat <<\'EOF\'\nFix the bug.\n\nAdd a mutex.\nEOF\n)"'
        self.assertEqual(GATE.commit_message(command), "Fix the bug.\n\nAdd a mutex.")
        command = "git commit -F - <<EOF\nFix the bug.\nEOF"
        self.assertEqual(GATE.commit_message(command), "Fix the bug.")
 
    def test_commit_text_in_data_is_not_a_commit(self):
        for command in (
            "cat > prompts.json <<'EOF'\n{\"p\": \"Write a git commit message\"}\nEOF",
            'echo "then run git commit and push"',
            "python3 -c \"print('git commit -m x')\"",
            "grep -r 'git commit' docs/",
            'git commit-tree HEAD -m "Fix the bug"',
        ):
            with self.subTest(command=command):
                self.assertIsNone(GATE.commit_message(command))
 
    def test_other_heredocs_stay_out_of_the_message(self):
        command = ("cat > notes.md <<'EOF'\n" + SLOP + "\nEOF\n"
                   'git add notes.md && git commit -m "Add the notes"')
        self.assertEqual(GATE.commit_message(command), "Add the notes")
 
    def test_no_message_commits_pass(self):
        for command in ("git commit --amend --no-edit", "git commit --fixup HEAD~1"):
            with self.subTest(command=command):
                self.assertIsNone(GATE.commit_message(command))
 
 
class HookTests(unittest.TestCase):
    def test_slop_commit_is_denied(self):
        out = run_hook('git commit -m "' + SLOP + '"')
        self.assertIn('"deny"', out)
 
    def test_slop_in_a_written_file_is_not_denied(self):
        command = ("cat > prompts.json <<'EOF'\n{\"p4\": \"Write a git commit message.\"}\nEOF\n"
                   "cat > run.sh <<'EOF'\n" + SLOP + "\nEOF")
        self.assertEqual(run_hook(command), "")
 
 
if __name__ == "__main__":
    unittest.main()
 

← Back to the episode