Usage grammar¶

The pattern node classes and low-level parser functions, exported because tools built on docopt reach into them. Most programs never touch them.

They are the parser's own vocabulary, not a promise: the drop-in guarantee covers docopt() and the mapping it returns, not these names, which follow docopt's unreleased master rather than the 0.6.2 spelling.

Use parse_tree for a friendlier entry point into the parse tree, and see the Usage DSL guide.

Pattern nodes¶

Pattern ¶

Base class for every node in a usage-pattern tree.

name `property` ¶

name: str | None

Key under which this element appears in the parsed result.

match ¶

match(
    left: list[Pattern],
    collected: list[Pattern] | None = None,
) -> MatchResult

Greedy single-result match: the first (greedy) outcome of matches.

Source code in src/docopt2/_parser.py

def match(self, left: list[Pattern], collected: list[Pattern] | None = None) -> MatchResult:
    """Greedy single-result match: the first (greedy) outcome of ``matches``."""
    collected = [] if collected is None else collected
    first = next(self.matches(left, collected), None)
    if first is None:
        return False, left, collected
    return True, first[0], first[1]

matches ¶

matches(
    left: list[Pattern], collected: list[Pattern]
) -> Iterator[MatchOutcome]

Yield every (remaining, collected) outcome of matching, greedy result first.

Unlike match (a single greedy result), this explores alternatives so a caller can find a fully-consuming match even when the greedy one leaves leaves over.

Source code in src/docopt2/_parser.py

def matches(self, left: list[Pattern], collected: list[Pattern]) -> Iterator[MatchOutcome]:
    """Yield every ``(remaining, collected)`` outcome of matching, greedy result first.

    Unlike ``match`` (a single greedy result), this explores alternatives so a caller can
    find a fully-consuming match even when the greedy one leaves leaves over.
    """
    raise NotImplementedError

flat ¶

flat(*types: type[Pattern]) -> list[Pattern]

Return the tree tips (or the nodes whose exact type is in types).

Source code in src/docopt2/_parser.py

def flat(self, *types: type[Pattern]) -> list[Pattern]:
    """Return the tree tips (or the nodes whose exact type is in ``types``)."""
    raise NotImplementedError

to_dict ¶

to_dict() -> dict[str, Any]

Return a JSON-serializable tree describing this pattern node.

Source code in src/docopt2/_parser.py

def to_dict(self) -> dict[str, Any]:
    """Return a JSON-serializable tree describing this pattern node."""
    raise NotImplementedError

fix ¶

fix() -> Pattern

Resolve shared identities and mark repeating elements. Returns self.

Source code in src/docopt2/_parser.py

def fix(self) -> Pattern:
    """Resolve shared identities and mark repeating elements. Returns self."""
    self.fix_identities()
    self.fix_repeating_arguments()
    return self

fix_identities ¶

fix_identities(
    uniq: dict[Pattern, Pattern] | None = None,
) -> None

Make equal tree tips point to the same object, so counts accumulate.

Source code in src/docopt2/_parser.py

def fix_identities(self, uniq: dict[Pattern, Pattern] | None = None) -> None:
    """Make equal tree tips point to the same object, so counts accumulate."""
    if not isinstance(self, BranchPattern):
        return
    # Map each distinct leaf to one canonical instance; the dict lookup resolves n leaves in O(n).
    canonical = {leaf: leaf for leaf in self.flat()} if uniq is None else uniq
    for index, child in enumerate(self.children):
        if isinstance(child, BranchPattern):
            child.fix_identities(canonical)
        else:
            self.children[index] = canonical[child]

fix_repeating_arguments ¶

fix_repeating_arguments() -> Pattern

Turn elements that may appear more than once into accumulators. Returns self.

Source code in src/docopt2/_parser.py

def fix_repeating_arguments(self) -> Pattern:
    """Turn elements that may appear more than once into accumulators. Returns self."""
    repeating = {leaf for leaf, count in _max_occurrences(self).items() if count > 1}
    # Materialize the targets before mutating: leaves hash by value, so mutating one would
    # corrupt set membership for an equal sibling still to be visited.
    for element in [leaf for leaf in self.flat() if leaf in repeating]:
        if type(element) is Argument or (isinstance(element, Option) and element.argcount):
            if element.value is None:
                element.value = []
            elif isinstance(element.value, str):
                element.value = element.value.split()
        if type(element) is Command or (isinstance(element, Option) and element.argcount == 0):
            element.value = 0
    return self

LeafPattern ¶

LeafPattern(name: str | None, value: LeafValue = None)

Leaf/terminal node of a pattern tree.

Source code in src/docopt2/_parser.py

def __init__(self, name: str | None, value: LeafValue = None) -> None:
    self._name = name
    # __eq__/__hash__ compare via repr, called in tight fix() loops; cache the repr and drop it
    # on value mutation so each compare stays O(1).
    self._cached_repr: str | None = None
    self._value = value

single_match ¶

single_match(left: list[Pattern]) -> SingleMatch

Find the first argument-vector leaf this pattern matches. Overridden per leaf.

Source code in src/docopt2/_parser.py

def single_match(self, left: list[Pattern]) -> SingleMatch:
    """Find the first argument-vector leaf this pattern matches. Overridden per leaf."""
    raise NotImplementedError

BranchPattern ¶

BranchPattern(*children: Pattern)

Branch/inner node of a pattern tree.

Source code in src/docopt2/_parser.py

def __init__(self, *children: Pattern) -> None:
    self.children = list(children)

Argument ¶

Argument(name: str | None, value: LeafValue = None)

A positional argument, written <name> or NAME in the usage pattern.

Source code in src/docopt2/_parser.py

def __init__(self, name: str | None, value: LeafValue = None) -> None:
    self._name = name
    # __eq__/__hash__ compare via repr, called in tight fix() loops; cache the repr and drop it
    # on value mutation so each compare stays O(1).
    self._cached_repr: str | None = None
    self._value = value

Command ¶

Command(name: str | None, value: bool = False)

A (sub)command literal, written as a bare word in the usage pattern.

Source code in src/docopt2/_parser.py

def __init__(self, name: str | None, value: bool = False) -> None:
    self._name = name
    self.value = value

Option ¶

Option(
    short: str | None = None,
    long: str | None = None,
    argcount: int = 0,
    value: LeafValue = False,
    env: str | None = None,
    config_key: str | None = None,
)

A short and/or long option, optionally taking a single argument.

Source code in src/docopt2/_parser.py

def __init__(
    self,
    short: str | None = None,
    long: str | None = None,
    argcount: int = 0,
    value: LeafValue = False,
    env: str | None = None,
    config_key: str | None = None,
) -> None:
    self.short = short
    self.long = long
    self.argcount = argcount
    self.value = None if value is False and argcount else value
    self.env = env  # `[env: VAR]` fallback source, resolved at parse time in docopt(), not here
    self.config_key = config_key  # `[config: dotted.key]` fallback, resolved against docopt(config=)

parse `classmethod` ¶

parse(option_description: str, source: str = '') -> Option

Build an Option from a single option-description line.

source is the full docstring. When given, a malformed line's error carries a caret pointing at the offending word within its reproduced source line.

Source code in src/docopt2/_parser.py

@classmethod
def parse(cls, option_description: str, source: str = "") -> Option:
    """Build an Option from a single option-description line.

    ``source`` is the full docstring. When given, a malformed line's error carries a caret
    pointing at the offending word within its reproduced source line.
    """
    short: str | None = None
    long: str | None = None
    argcount = 0
    value: LeafValue = False
    stripped = option_description.strip()
    options, _, description = stripped.partition("  ")
    options = options.replace(",", " ").replace("=", " ")
    flag_tokens = 0
    argument_tokens = 0
    for token in options.split():
        if token.startswith("--"):
            long = token
            flag_tokens += 1
        elif token.startswith("-"):
            short = token
            flag_tokens += 1
        else:
            argcount = 1
            argument_tokens += 1
    if argument_tokens > flag_tokens:
        # Each flag takes at most one arg name, so more argument words than flags almost always
        # means the option and its description were run together with a single space; name the
        # real cause (caret under the first stray word) instead of a cryptic later "unmatched".
        offending = next(match for match in re.finditer(r"\S+", options) if not match.group().startswith("-"))
        base = source.find(stripped)
        carets = [Caret(base + offending.start(), base + offending.end(), "read as an argument name")]
        snippets = [Snippet(source, "in the options:", carets)] if base != -1 else []
        diagnostic = Diagnostic(
            summary=f"option `{stripped}` has more argument words than flags",
            snippets=snippets,
            help="separate the option from its description with at least two spaces",
        )
        raise DocoptLanguageError(diagnostic.render())
    env_match = _ENV_PATTERN.search(description)
    env = env_match.group(1) if env_match else None
    config_match = _CONFIG_PATTERN.search(description)
    config_key = config_match.group(1) if config_match else None
    if argcount:
        # Strip `[env:]`/`[config:]` first so the greedy `[default: (.*)]` cannot swallow them.
        without_sources = _CONFIG_PATTERN.sub("", _ENV_PATTERN.sub("", description))
        matched = _DEFAULT_PATTERN.findall(without_sources)
        value = matched[0] if matched else None
    return cls(short, long, argcount, value, env, config_key)

Required ¶

Required(*children: Pattern)

All children must match, in order.

Source code in src/docopt2/_parser.py

def __init__(self, *children: Pattern) -> None:
    self.children = list(children)

Optional ¶

Optional(*children: Pattern)

Children may match. A non-match is not a failure.

Source code in src/docopt2/_parser.py

def __init__(self, *children: Pattern) -> None:
    self.children = list(children)

OptionsShortcut ¶

OptionsShortcut(*children: Pattern)

Marker/placeholder for the [options] shortcut.

Source code in src/docopt2/_parser.py

def __init__(self, *children: Pattern) -> None:
    self.children = list(children)

Either ¶

Either(*children: Pattern)

Exactly one branch must match. The one leaving the fewest leaves wins.

Source code in src/docopt2/_parser.py

def __init__(self, *children: Pattern) -> None:
    self.children = list(children)

OneOrMore ¶

OneOrMore(*children: Pattern)

The single child must match one or more times.

Source code in src/docopt2/_parser.py

def __init__(self, *children: Pattern) -> None:
    self.children = list(children)

matches ¶

matches(
    left: list[Pattern], collected: list[Pattern]
) -> Iterator[MatchOutcome]

Greedy-first: the longest run of the child comes out first, then progressively shorter ones.

Source code in src/docopt2/_parser.py

def matches(self, left: list[Pattern], collected: list[Pattern]) -> Iterator[MatchOutcome]:
    """Greedy-first: the longest run of the child comes out first, then progressively shorter ones."""
    # Iterative on an explicit stack, not recursive: a repetition consumes one token per round, so a
    # recursive walk would nest once per argv token, and `prog <files>...` over a shell glob of a few
    # thousand files is ordinary use the original parses (its OneOrMore is a `while` loop).
    # Each frame holds the outcome that opened it and yields it once its own run is spent, which is what
    # puts the deepest (greediest) run first.
    child = self.children[0]
    stack: list[tuple[Iterator[MatchOutcome], int, MatchOutcome | None]] = [
        (child.matches(left, collected), len(left), None)
    ]
    while stack:
        outcomes, remaining_count, opener = stack[-1]
        outcome = next(outcomes, None)
        if outcome is None:  # this run is spent; the outcome that opened it comes after everything it led to
            stack.pop()
            if opener is not None:
                yield opener
        elif len(outcome[0]) < remaining_count:  # progress: try to repeat before settling for this run
            stack.append((child.matches(*outcome), len(outcome[0]), outcome))
        else:  # matched without consuming (`[NAME]...` with nothing left): repeating would not terminate
            yield outcome

Tokens ¶

Tokens(
    source: list[str] | tuple[str, ...] | str,
    error: ErrorType = DocoptExit,
    *,
    spans: list[Span] | None = None,
    text: str = "",
    usage: str = "",
    exit_code: int = 1,
)

A mutable token stream that remembers which error class to raise and each token's span.

Source code in src/docopt2/_parser.py

def __init__(
    self,
    source: list[str] | tuple[str, ...] | str,
    error: ErrorType = DocoptExit,
    *,
    spans: list[Span] | None = None,
    text: str = "",
    usage: str = "",
    exit_code: int = 1,
) -> None:
    super().__init__(source.split() if isinstance(source, str) else source)
    self.error = error
    self.text = text  # the full source string, for caret rendering
    self.spans: list[Span] = spans if spans is not None else [None] * len(self)
    self._last_span: Span = None
    self.usage = usage  # threaded onto a DocoptExit so argv errors carry usage/exit_code too
    self.exit_code = exit_code

parsing_argv `property` ¶

parsing_argv: bool

Whether this stream is argv (a user error, DocoptExit) rather than the docstring.

from_pattern `staticmethod` ¶

from_pattern(source: str) -> Tokens

Tokenize a usage-pattern string (errors become DocoptLanguageError).

Source code in src/docopt2/_parser.py

@staticmethod
def from_pattern(source: str) -> Tokens:
    """Tokenize a usage-pattern string (errors become DocoptLanguageError)."""
    spanned = [(match.group(), match.start(), match.end()) for match in _PATTERN_TOKEN.finditer(source)]
    _check_brackets(source, spanned)
    texts = [text for text, _, _ in spanned]
    spans: list[Span] = [(start, end) for _, start, end in spanned]
    return Tokens(texts, error=DocoptLanguageError, spans=spans, text=source)

move ¶

move() -> str | None

Pop and return the next token, or None if the stream is empty.

Source code in src/docopt2/_parser.py

def move(self) -> str | None:
    """Pop and return the next token, or None if the stream is empty."""
    if not len(self):
        return None
    self._last_span = self.spans.pop(0)
    return self.pop(0)

current ¶

current() -> str | None

Return the next token without consuming it, or None if empty.

Source code in src/docopt2/_parser.py

def current(self) -> str | None:
    """Return the next token without consuming it, or None if empty."""
    return self[0] if len(self) else None

current_span ¶

current_span() -> Span

The source span of the next token, if any.

Source code in src/docopt2/_parser.py

def current_span(self) -> Span:
    """The source span of the next token, if any."""
    return self.spans[0] if self.spans else None

fail ¶

fail(message: str) -> DocoptExit | DocoptLanguageError

Build the error as a diagnostic, with a caret at the current (or last consumed) token.

Source code in src/docopt2/_parser.py

def fail(self, message: str) -> DocoptExit | DocoptLanguageError:
    """Build the error as a diagnostic, with a caret at the current (or last consumed) token."""
    span = self.current_span() or self._last_span
    intro = "in the arguments:" if self.parsing_argv else "in the usage:"
    carets = [Caret(span[0], span[1], "here")] if span is not None else []
    snippets = [Snippet(self.text, intro, carets)] if self.text else []
    diagnostic = Diagnostic(summary=message, snippets=snippets)
    if self.parsing_argv:
        return DocoptExit(diagnostic=diagnostic, usage=self.usage, exit_code=self.exit_code)
    return self.error(diagnostic.render())

Parser functions¶

formal_usage ¶

formal_usage(section: str) -> str

Rewrite a usage section into a formal pattern with the program name as separator.

Source code in src/docopt2/_parser.py

def formal_usage(section: str) -> str:
    """Rewrite a usage section into a formal pattern with the program name as separator."""
    _, _, body = section.partition(":")
    words = body.split()
    if not words:
        raise DocoptLanguageError(Diagnostic(summary="the usage section names no program").render())
    program = words[0]
    return "( " + " ".join(") | (" if word == program else word for word in words[1:]) + " )"

parse_section ¶

parse_section(name: str, source: str) -> list[str]

Return the stripped text of every section whose header contains name.

Source code in src/docopt2/_parser.py

def parse_section(name: str, source: str) -> list[str]:
    """Return the stripped text of every section whose header contains ``name``."""
    return [section.strip() for section in _section_pattern(name).findall(source)]

parse_defaults ¶

parse_defaults(doc: str) -> list[Option]

Collect option defaults from every options: section of the docstring.

Source code in src/docopt2/_parser.py

def parse_defaults(doc: str) -> list[Option]:
    """Collect option defaults from every ``options:`` section of the docstring."""
    return [Option.parse(text, doc) for text in _option_chunks(doc) if text.startswith("-")]

parse_pattern ¶

parse_pattern(
    source: str | Tokens, options: list[Option]
) -> Required

Parse a formal usage string (or pre-spanned Tokens) into a Required pattern tree.

Source code in src/docopt2/_parser.py

def parse_pattern(source: str | Tokens, options: list[Option]) -> Required:
    """Parse a formal usage string (or pre-spanned ``Tokens``) into a Required pattern tree."""
    tokens = source if isinstance(source, Tokens) else Tokens.from_pattern(source)
    result = parse_expr(tokens, options)
    return Required(*result)

parse_argv ¶

parse_argv(
    tokens: Tokens,
    options: list[Option],
    options_first: bool = False,
    negative_numbers: bool = False,
    allow_abbrev: bool = True,
) -> list[Pattern]

Parse the command-line argument vector.

Parameters:

Name	Type	Description	Default
`tokens`	`Tokens`	The argument vector to parse.	required
`options`	`list[Option]`	The options declared by the usage message.	required
`options_first`	`bool`	Require options to precede positionals, narrowing the grammar to `argv ::= [ long \| shorts ]* [ argument ]* [ '--' [ argument ]* ] ;`. Otherwise options and positionals may intermix.	`False`
`negative_numbers`	`bool`	Treat a token like `-3` or `-6.28` as a positional argument rather than a cluster of short options.	`False`
`allow_abbrev`	`bool`	When False, a long option must be written in full, with no `--ver` -> `--version` de-abbreviation.	`True`

Source code in src/docopt2/_parser.py

def parse_argv(
    tokens: Tokens,
    options: list[Option],
    options_first: bool = False,
    negative_numbers: bool = False,
    allow_abbrev: bool = True,
) -> list[Pattern]:
    """Parse the command-line argument vector.

    Args:
        tokens: The argument vector to parse.
        options: The options declared by the usage message.
        options_first: Require options to precede positionals, narrowing the grammar to
            ``argv ::= [ long | shorts ]* [ argument ]* [ '--' [ argument ]* ] ;``. Otherwise
            options and positionals may intermix.
        negative_numbers: Treat a token like ``-3`` or ``-6.28`` as a positional argument
            rather than a cluster of short options.
        allow_abbrev: When False, a long option must be written in full, with no
            ``--ver`` -> ``--version`` de-abbreviation.
    """
    parsed: list[Pattern] = []
    current = tokens.current()
    while current is not None:
        if current == "--":
            return parsed + [Argument(None, value) for value in tokens]
        if current.startswith("--"):
            parsed += parse_long(tokens, options, allow_abbrev)
        elif current.startswith("-") and current != "-" and not (negative_numbers and _is_number(current)):
            parsed += parse_shorts(tokens, options)
        elif options_first:
            return parsed + [Argument(None, value) for value in tokens]
        else:
            parsed.append(Argument(None, tokens.move()))
        current = tokens.current()
    return parsed

transform ¶

transform(pattern: Pattern) -> Either

Expand a pattern into an (almost) equivalent one with a single top-level Either.

Example: ((-a | -b) (-c | -d)) becomes (-a -c | -a -d | -b -c | -b -d). Quirks: [-a] becomes (-a), and (-a...) becomes (-a -a).

Source code in src/docopt2/_parser.py

def transform(pattern: Pattern) -> Either:
    """Expand a pattern into an (almost) equivalent one with a single top-level Either.

    Example: ``((-a | -b) (-c | -d))`` becomes ``(-a -c | -a -d | -b -c | -b -d)``.
    Quirks: ``[-a]`` becomes ``(-a)``, and ``(-a...)`` becomes ``(-a -a)``.
    """
    result: list[list[Pattern]] = []
    groups: list[list[Pattern]] = [[pattern]]
    branch_types = (Required, Optional, OptionsShortcut, Either, OneOrMore)
    while groups:
        children = groups.pop(0)
        branches = [child for child in children if isinstance(child, branch_types)]
        if not branches:
            result.append(children)
            continue
        branch = branches[0]
        children.remove(branch)
        if isinstance(branch, Either):
            groups.extend([branch_child, *children] for branch_child in branch.children)
        elif isinstance(branch, OneOrMore):
            groups.append(branch.children * 2 + children)
        else:
            groups.append(branch.children + children)
    return Either(*[Required(*case) for case in result])

Usage grammar¶

Pattern nodes¶

Pattern ¶

name property ¶

match ¶

matches ¶

flat ¶

to_dict ¶

fix ¶

fix_identities ¶

fix_repeating_arguments ¶

LeafPattern ¶

single_match ¶

BranchPattern ¶

Argument ¶

Command ¶

Option ¶

parse classmethod ¶

Required ¶

Optional ¶

OptionsShortcut ¶

Either ¶

OneOrMore ¶

matches ¶

Tokens ¶

parsing_argv property ¶

from_pattern staticmethod ¶

move ¶

current ¶

current_span ¶

fail ¶

Parser functions¶

formal_usage ¶

parse_section ¶

parse_defaults ¶

parse_pattern ¶

parse_argv ¶

transform ¶

name `property` ¶

parse `classmethod` ¶

parsing_argv `property` ¶

from_pattern `staticmethod` ¶