update qute-pass

config/qutebrowser/greasemonkey/qute-pass

@@ -40,11 +40,13 @@ import argparse
 import enum
 import fnmatch
 import functools
+import idna
 import os
 import re
 import shlex
 import subprocess
 import sys
+import unicodedata
 from urllib.parse import urlparse
 
 import tldextract
@@ -54,42 +56,102 @@ def expanded_path(path):
     # Expand potential ~ in paths, since this script won't be called from a shell that does it for us
     expanded = os.path.expanduser(path)
     # Add trailing slash if not present
-    return os.path.join(expanded, '')
+    return os.path.join(expanded, "")
 
 
-argument_parser = argparse.ArgumentParser(description=__doc__, usage=USAGE, epilog=EPILOG)
+argument_parser = argparse.ArgumentParser(
-argument_parser.add_argument('url', nargs='?', default=os.getenv('QUTE_URL'))
+    description=__doc__, usage=USAGE, epilog=EPILOG
-argument_parser.add_argument('--password-store', '-p',
+)
-                             default=expanded_path(os.getenv('PASSWORD_STORE_DIR', default='~/.password-store')),
+argument_parser.add_argument("url", nargs="?", default=os.getenv("QUTE_URL"))
-                             help='Path to your pass password-store (only used in pass-mode)', type=expanded_path)
+argument_parser.add_argument(
-argument_parser.add_argument('--mode', '-M', choices=['pass', 'gopass'], default="pass",
+    "--password-store",
-                             help='Select mode [gopass] to use gopass instead of the standard pass.')
+    "-p",
-argument_parser.add_argument('--prefix', type=str,
+    default=expanded_path(os.getenv("PASSWORD_STORE_DIR", default="~/.password-store")),
-                             help='Search only the given subfolder of the store (only used in gopass-mode)')
+    help="Path to your pass password-store (only used in pass-mode)",
-argument_parser.add_argument('--username-pattern', '-u', default=r'.*/(.+)',
+    type=expanded_path,
-                             help='Regular expression that matches the username')
+)
-argument_parser.add_argument('--username-target', '-U', choices=['path', 'secret'], default='path',
+argument_parser.add_argument(
-                             help='The target for the username regular expression')
+    "--mode",
-argument_parser.add_argument('--password-pattern', '-P', default=r'(.*)',
+    "-M",
-                             help='Regular expression that matches the password')
+    choices=["pass", "gopass"],
-argument_parser.add_argument('--dmenu-invocation', '-d', default='rofi -dmenu',
+    default="pass",
-                             help='Invocation used to execute a dmenu-provider')
+    help="Select mode [gopass] to use gopass instead of the standard pass.",
-argument_parser.add_argument('--no-insert-mode', '-n', dest='insert_mode', action='store_false',
+)
-                             help="Don't automatically enter insert mode")
+argument_parser.add_argument(
-argument_parser.add_argument('--io-encoding', '-i', default='UTF-8',
+    "--prefix",
-                             help='Encoding used to communicate with subprocesses')
+    type=str,
-argument_parser.add_argument('--merge-candidates', '-m', action='store_true',
+    help="Search only the given subfolder of the store (only used in gopass-mode)",
-                             help='Merge pass candidates for fully-qualified and registered domain name')
+)
-argument_parser.add_argument('--extra-url-suffixes', '-s', default='',
+argument_parser.add_argument(
-                             help='Comma-separated string containing extra suffixes (e.g local)')
+    "--username-pattern",
-argument_parser.add_argument('--unfiltered', dest='unfiltered', action='store_true',
+    "-u",
-                             help='Show an unfiltered selection of all passwords in the store')
+    default=r".*/(.+)",
-argument_parser.add_argument('--always-show-selection', dest='always_show_selection', action='store_true',
+    help="Regular expression that matches the username",
-                             help='Always show selection, even if there is only a single match')
+)
+argument_parser.add_argument(
+    "--username-target",
+    "-U",
+    choices=["path", "secret"],
+    default="path",
+    help="The target for the username regular expression",
+)
+argument_parser.add_argument(
+    "--password-pattern",
+    "-P",
+    default=r"(.*)",
+    help="Regular expression that matches the password",
+)
+argument_parser.add_argument(
+    "--dmenu-invocation",
+    "-d",
+    default="rofi -dmenu",
+    help="Invocation used to execute a dmenu-provider",
+)
+argument_parser.add_argument(
+    "--no-insert-mode",
+    "-n",
+    dest="insert_mode",
+    action="store_false",
+    help="Don't automatically enter insert mode",
+)
+argument_parser.add_argument(
+    "--io-encoding",
+    "-i",
+    default="UTF-8",
+    help="Encoding used to communicate with subprocesses",
+)
+argument_parser.add_argument(
+    "--merge-candidates",
+    "-m",
+    action="store_true",
+    help="Merge pass candidates for fully-qualified and registered domain name",
+)
+argument_parser.add_argument(
+    "--extra-url-suffixes",
+    "-s",
+    default="",
+    help="Comma-separated string containing extra suffixes (e.g local)",
+)
+argument_parser.add_argument(
+    "--unfiltered",
+    dest="unfiltered",
+    action="store_true",
+    help="Show an unfiltered selection of all passwords in the store",
+)
+argument_parser.add_argument(
+    "--always-show-selection",
+    dest="always_show_selection",
+    action="store_true",
+    help="Always show selection, even if there is only a single match",
+)
 group = argument_parser.add_mutually_exclusive_group()
-group.add_argument('--username-only', '-e', action='store_true', help='Only insert username')
+group.add_argument(
-group.add_argument('--password-only', '-w', action='store_true', help='Only insert password')
+    "--username-only", "-e", action="store_true", help="Only insert username"
-group.add_argument('--otp-only', '-o', action='store_true', help='Only insert OTP code')
+)
+group.add_argument(
+    "--password-only", "-w", action="store_true", help="Only insert password"
+)
+group.add_argument("--otp-only", "-o", action="store_true", help="Only insert OTP code")
 
 stderr = functools.partial(print, file=sys.stderr)
 
@@ -112,11 +174,30 @@ class CouldNotMatchPassword(Exception):
 
 
 def qute_command(command):
-    with open(os.environ['QUTE_FIFO'], 'w') as fifo:
+    with open(os.environ["QUTE_FIFO"], "w") as fifo:
-        fifo.write(command + '\n')
+        fifo.write(command + "\n")
         fifo.flush()
 
 
+# Encode candidate string parts as Internationalized Domain Name, doing
+# Unicode normalization before. This allows to properly match (non-ASCII)
+# pass entries with the corresponding domain names.
+def idna_encode(name):
+    # Do Unicode normalization first, we use form NFKC because:
+    # 1. Use the compatibility normalization because these sequences have "the same meaning in some contexts"
+    # 2. idna.encode() below requires the Unicode strings to be in normalization form C
+    # See https://en.wikipedia.org/wiki/Unicode_equivalence#Normal_forms
+    unicode_normalized = unicodedata.normalize("NFKC", name)
+    # Empty strings can not be encoded, they appear for example as empty
+    # parts in split_path. If something like this happens, we just fall back
+    # to the unicode representation (which may already be ASCII then).
+    try:
+        idna_encoded = idna.encode(unicode_normalized)
+    except idna.IDNAError:
+        idna_encoded = unicode_normalized
+    return idna_encoded
+
+
 def find_pass_candidates(domain, unfiltered=False):
     candidates = []
 
@@ -124,25 +205,37 @@ def find_pass_candidates(domain, unfiltered=False):
         gopass_args = ["gopass", "list", "--flat"]
         if arguments.prefix:
             gopass_args.append(arguments.prefix)
-        all_passwords = subprocess.run(gopass_args, stdout=subprocess.PIPE).stdout.decode("UTF-8").splitlines()
+        all_passwords = (
+            subprocess.run(gopass_args, stdout=subprocess.PIPE)
+            .stdout.decode("UTF-8")
+            .splitlines()
+        )
 
         for password in all_passwords:
             if unfiltered or domain in password:
                 candidates.append(password)
     else:
-        for path, directories, file_names in os.walk(arguments.password_store, followlinks=True):
+        idna_domain = idna_encode(domain)
-            secrets = fnmatch.filter(file_names, '*.gpg')
+        for path, directories, file_names in os.walk(
+            arguments.password_store, followlinks=True
+        ):
+            secrets = fnmatch.filter(file_names, "*.gpg")
             if not secrets:
                 continue
 
             # Strip password store path prefix to get the relative pass path
-            pass_path = path[len(arguments.password_store):]
+            pass_path = path[len(arguments.password_store) :]
             split_path = pass_path.split(os.path.sep)
+            idna_split_path = [idna_encode(part) for part in split_path]
             for secret in secrets:
                 secret_base = os.path.splitext(secret)[0]
-                if not unfiltered and domain not in (split_path + [secret_base]):
+                idna_secret_base = idna_encode(secret_base)
+                if not unfiltered and idna_domain not in (
+                    idna_split_path + [idna_secret_base]
+                ):
                     continue
 
+                # Append the unencoded Unicode path/name since this is how pass uses them
                 candidates.append(os.path.join(pass_path, secret_base))
     return candidates
 
@@ -151,32 +244,38 @@ def _run_pass(pass_arguments):
     # The executable is conveniently named after it's mode [pass|gopass].
     pass_command = [arguments.mode]
     env = os.environ.copy()
-    env['PASSWORD_STORE_DIR'] = arguments.password_store
+    env["PASSWORD_STORE_DIR"] = arguments.password_store
-    process = subprocess.run(pass_command + pass_arguments, env=env, stdout=subprocess.PIPE)
+    process = subprocess.run(
+        pass_command + pass_arguments, env=env, stdout=subprocess.PIPE
+    )
     return process.stdout.decode(arguments.io_encoding).strip()
 
 
 def pass_(path):
-    return _run_pass(['show', path])
+    return _run_pass(["show", path])
 
 
 def pass_otp(path):
     if arguments.mode == "gopass":
-        return _run_pass(['otp', '-o', path])
+        return _run_pass(["otp", "-o", path])
-    return _run_pass(['otp', path])
+    return _run_pass(["otp", path])
 
 
 def dmenu(items, invocation):
     command = shlex.split(invocation)
-    process = subprocess.run(command, input='\n'.join(items).encode(arguments.io_encoding), stdout=subprocess.PIPE)
+    process = subprocess.run(
+        command,
+        input="\n".join(items).encode(arguments.io_encoding),
+        stdout=subprocess.PIPE,
+    )
     return process.stdout.decode(arguments.io_encoding).strip()
 
 
 def fake_key_raw(text):
     for character in text:
         # Escape all characters by default, space requires special handling
-        sequence = '" "' if character == ' ' else r'\{}'.format(character)
+        sequence = '" "' if character == " " else r"\{}".format(character)
-        qute_command('fake-key {}'.format(sequence))
+        qute_command("fake-key {}".format(sequence))
 
 
 def extract_password(secret, pattern):
@@ -186,7 +285,9 @@ def extract_password(secret, pattern):
     try:
         return match.group(1)
     except IndexError:
-        raise CouldNotMatchPassword("Pattern did not contain capture group, please use capture group. Example: (.*)")
+        raise CouldNotMatchPassword(
+            "Pattern did not contain capture group, please use capture group. Example: (.*)"
+        )
 
 
 def extract_username(target, pattern):
@@ -196,7 +297,9 @@ def extract_username(target, pattern):
     try:
         return match.group(1)
     except IndexError:
-        raise CouldNotMatchUsername("Pattern did not contain capture group, please use capture group. Example: (.*)")
+        raise CouldNotMatchUsername(
+            "Pattern did not contain capture group, please use capture group. Example: (.*)"
+        )
 
 
 def main(arguments):
@@ -204,7 +307,9 @@ def main(arguments):
         argument_parser.print_help()
         return ExitCodes.FAILURE
 
-    extractor = tldextract.TLDExtract(extra_suffixes=arguments.extra_url_suffixes.split(','))
+    extractor = tldextract.TLDExtract(
+        extra_suffixes=arguments.extra_url_suffixes.split(",")
+    )
     extract_result = extractor(arguments.url)
 
     # Try to find candidates using targets in the following order: fully-qualified domain name (includes subdomains),
@@ -213,16 +318,30 @@ def main(arguments):
     candidates = set()
     attempted_targets = []
 
-    private_domain = ''
+    private_domain = ""
     if not extract_result.suffix:
-        private_domain = ('.'.join((extract_result.subdomain, extract_result.domain))
+        private_domain = (
-                          if extract_result.subdomain else extract_result.domain)
+            ".".join((extract_result.subdomain, extract_result.domain))
+            if extract_result.subdomain
+            else extract_result.domain
+        )
 
     netloc = urlparse(arguments.url).netloc
 
-    for target in filter(None, [extract_result.fqdn, extract_result.registered_domain, extract_result.ipv4, private_domain, netloc]):
+    for target in filter(
+        None,
+        [
+            extract_result.fqdn,
+            extract_result.registered_domain,
+            extract_result.ipv4,
+            private_domain,
+            netloc,
+        ],
+    ):
         attempted_targets.append(target)
-        target_candidates = find_pass_candidates(target, unfiltered=arguments.unfiltered)
+        target_candidates = find_pass_candidates(
+            target, unfiltered=arguments.unfiltered
+        )
         if not target_candidates:
             continue
 
@@ -231,7 +350,11 @@ def main(arguments):
             break
     else:
         if not candidates:
-            stderr('No pass candidates for URL {!r} found! (I tried {!r})'.format(arguments.url, attempted_targets))
+            stderr(
+                "No pass candidates for URL {!r} found! (I tried {!r})".format(
+                    arguments.url, attempted_targets
+                )
+            )
             return ExitCodes.NO_PASS_CANDIDATES
 
     if len(candidates) == 1 and not arguments.always_show_selection:
@@ -246,9 +369,12 @@ def main(arguments):
     # If username-target is path and user asked for username-only, we don't need to run pass.
     # Or if using otp-only, it will run pass on its own.
     secret = None
-    if not (arguments.username_target == 'path' and arguments.username_only) and not arguments.otp_only:
+    if (
+        not (arguments.username_target == "path" and arguments.username_only)
+        and not arguments.otp_only
+    ):
         secret = pass_(selection)
-    username_target = selection if arguments.username_target == 'path' else secret
+    username_target = selection if arguments.username_target == "path" else secret
     try:
         if arguments.username_only:
             fake_key_raw(extract_username(username_target, arguments.username_pattern))
@@ -261,21 +387,25 @@ def main(arguments):
             # Enter username and password using fake-key and <Tab> (which seems to work almost universally), then switch
             # back into insert-mode, so the form can be directly submitted by hitting enter afterwards
             fake_key_raw(extract_username(username_target, arguments.username_pattern))
-            qute_command('fake-key <Tab>')
+            qute_command("fake-key <Tab>")
             fake_key_raw(extract_password(secret, arguments.password_pattern))
     except CouldNotMatchPassword as e:
-        stderr('Failed to match password, target: secret, error: {}'.format(e))
+        stderr("Failed to match password, target: secret, error: {}".format(e))
         return ExitCodes.COULD_NOT_MATCH_PASSWORD
     except CouldNotMatchUsername as e:
-        stderr('Failed to match username, target: {}, error: {}'.format(arguments.username_target, e))
+        stderr(
+            "Failed to match username, target: {}, error: {}".format(
+                arguments.username_target, e
+            )
+        )
         return ExitCodes.COULD_NOT_MATCH_USERNAME
 
     if arguments.insert_mode:
-        qute_command('mode-enter insert')
+        qute_command("mode-enter insert")
 
     return ExitCodes.SUCCESS
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     arguments = argument_parser.parse_args()
     sys.exit(main(arguments))