Make URLs, etc. configurable

Playright needs to be updated frequently in order to update its Firefox
build.  The Chase website has a very strict browser support policy, and
frequently drops support for old Firefox versions.

.containerignore

@@ -0,0 +1,5 @@
+*
+!.git
+!pinentry-stub.sh
+!pyproject.toml
+!xactfetch.py

Containerfile

@@ -0,0 +1,76 @@
+FROM registry.fedoraproject.org/fedora-minimal:38 AS build
+
+RUN --mount=type=cache,target=/var/cache \
+	microdnf install -y \
+		--setopt install_weak_deps=0 \
+		rust \
+		cargo \
+	&& :
+
+RUN cargo install rbw
+
+RUN --mount=type=cache,target=/var/cache \
+	microdnf install -y \
+		--setopt install_weak_deps=0 \
+		git-core \
+		python3-devel \
+		python3-pip \
+		python3-wheel \
+	&& :
+
+COPY . /src
+
+RUN python3 -m pip wheel -w /wheels /src
+
+FROM registry.fedoraproject.org/fedora-minimal:38
+
+RUN --mount=type=cache,target=/var/cache \
+	microdnf install -y \
+		--setopt install_weak_deps=0 \
+		alsa-lib \
+		atk \
+		cairo \
+		cairo-gobject \
+		dbus-glib \
+		fontconfig \
+		freetype \
+		gdk-pixbuf2 \
+		gtk3 \
+		libX11 \
+		libX11-xcb \
+		libXcomposite \
+		libXcursor \
+		libXdamage \
+		libXext \
+		libXfixes \
+		libXi \
+		libXrandr \
+		libXrender \
+		libXtst \
+		libxcb \
+		pango \
+		python3 \
+		python3-pip \
+		tini \
+	&& echo xactfetch:x:2468: >> /etc/group \
+	&& echo xactfetch:*:2468:2468:xactfetch:/var/lib/xactfetch:/sbin/nologin >> /etc/passwd \
+	&& :
+
+ENV PLAYWRIGHT_BROWSERS_PATH=/usr/local/playwright/browsers
+
+RUN --mount=type=bind,from=build,source=/,target=/build \
+	python3 -m pip install --no-index -f /build/wheels xactfetch \
+	&& cp /build/root/.cargo/bin/rbw* /usr/local/bin/ \
+	&& install /build/src/pinentry-stub.sh /usr/local/bin/pinentry-stub \
+	&& playwright install firefox \
+	&& :
+
+VOLUME /var/lib/xactfetch
+
+WORKDIR /var/lib/xactfetch
+
+USER 2468:2468
+
+ENV XDG_CONFIG_HOME=/etc
+
+ENTRYPOINT ["tini", "xactfetch", "--"]

pinentry-stub.sh

@@ -0,0 +1,19 @@
+#!/bin/sh
+# vim: set sw=4 ts=4 sts=4 et :
+
+while IFS=' ' read -r cmd args; do
+    case "${cmd}" in
+    GETPIN)
+        printf 'D %s\n' "$(cat "${PINENTRY_PASSWORD_FILE}")"
+        ;;
+    SETPROMPT|SETTITLE|SETDESC)
+        echo OK
+        ;;
+    BYE)
+        exit 0
+        ;;
+    *)
+        printf 'ERR Unknonw command\n'
+        ;;
+    esac
+done

pyproject.toml

@@ -11,7 +11,7 @@ classifiers = [
     "Programming Language :: Python :: 3",
 ]
 dependencies = [
-    "playwright~=1.32.1",
+    "playwright~=1.32",
     "requests~=2.29.0",
 ]
 dynamic = ["version"]
@@ -23,6 +23,8 @@ xactfetch = "xactfetch:main"
 requires = ["setuptools", "setuptools-scm"]
 build-backend = "setuptools.build_meta"
 
+[tool.setuptools_scm]
+
 [tool.pyright]
 venvPath = '.'
 venv = '.venv'

xactfetch.container

@@ -0,0 +1,23 @@
+[Unit]
+Description=Fetch transaction lists from bank websites
+
+[Service]
+Type=oneshot
+Restart=on-failure
+RestartSec=2h
+
+[Container]
+Image=git.pyrocufflink.net/packages/xactfetch
+Environment=PINENTRY_PASSWORD_FILE=/run/secrets/xactfetch
+Environment=FIREFLY_IMPORT_SECRET_FILE=/run/secrets/firefly-import-secret
+Environment=FIREFLY_IMPORT_PASSWORD_FILE=/run/secrets/firefly-import-password
+Environment=FIREFLY_IMPORT_USER=svc.xactfetch
+Environment=FIREFLY_III_URL=https://firefly.pyrocufflink.blue
+Environment=FIREFLY_IMPORT_URL=https://firefly-importer.pyrocufflink.blue
+Environment=NTFY_URL=https://ntfy.pyrocufflink.net
+Environment=NTFY_TOPIC=dustin
+Secret=xactfetch
+Secret=firefly-import-password
+Secret=firefly-import-secret
+Volume=xactfetch:/var/lib/xactfetch:U
+Volume=%E/rbw-%N/config.json:/etc/rbw/config.json:ro,z

xactfetch.py

@@ -1,14 +1,12 @@
 import base64
 import copy
 import datetime
+import getpass
 import json
 import logging
 import os
 import random
-import shlex
-import shutil
 import subprocess
-import tempfile
 import urllib.parse
 from pathlib import Path
 from types import TracebackType
@@ -22,9 +20,11 @@ from playwright.sync_api import sync_playwright
 log = logging.getLogger('xactfetch')
 
 
-NTFY_URL = 'https://ntfy.pyrocufflink.net'
+NTFY_URL = os.environ['NTFY_URL']
-NTFY_TOPIC = 'dustin'
+NTFY_TOPIC = os.environ['NTFY_TOPIC']
-FIREFLY_III_URL = 'https://firefly.pyrocufflink.blue'
+FIREFLY_III_URL = os.environ['FIREFLY_III_URL']
+FIREFLY_III_IMPORTER_URL = os.environ['FIREFLY_IMPORT_URL']
+
 ACCOUNTS = {
     'commerce': {
         '8357': 1,
@@ -34,6 +34,42 @@ ACCOUNTS = {
 }
 
 
+class FireflyImporter:
+    def __init__(
+        self,
+        url: str,
+        secret: str,
+        auth: Optional[tuple[str, str]],
+    ) -> None:
+        self.url = url
+        self.secret = secret
+        self.auth = auth
+
+    def import_csv(
+        self,
+        csv: Path,
+        config: dict[str, Any],
+    ) -> None:
+        log.debug('Importing transactions from %s to Firefly III', csv)
+        url = f'{self.url.rstrip("/")}/autoupload'
+        with csv.open('rb') as f:
+            r = requests.post(
+                url,
+                auth=self.auth,
+                headers={
+                    'Accept': 'application/json',
+                },
+                params={
+                    'secret': self.secret,
+                },
+                files={
+                    'importable': ('import.csv', f),
+                    'json': ('import.json', json.dumps(config)),
+                },
+            )
+            r.raise_for_status()
+
+
 def ntfy(
     message: Optional[str] = None,
     topic: str = NTFY_TOPIC,
@@ -132,48 +168,11 @@ def rfc2047_base64encode(
     return f"=?UTF-8?B?{encoded}?="
 
 
-def firefly_import(csv: Path, config: dict[str, Any], token: str) -> None:
+def secret_from_file(env: str, default: str) -> str:
-    log.debug('Importing transactions from %s to Firefly III', csv)
+    filename = os.environ.get(env, default)
-    env = {
+    log.debug('Loading secret value from %s', filename)
-        'PATH': os.environ['PATH'],
+    with open(filename, 'r', encoding='utf-8') as f:
-        'FIREFLY_III_ACCESS_TOKEN': token,
+        return f.read().rstrip()
-        'IMPORT_DIR_ALLOWLIST': '/import',
-        'FIREFLY_III_URL': FIREFLY_III_URL,
-        'WEB_SERVER': 'false',
-    }
-    with tempfile.TemporaryDirectory() as tmpdir:
-        dest = Path(tmpdir) / 'import.csv'
-        log.debug('Copying %s to %s', csv, dest)
-        shutil.copyfile(csv, dest)
-        configfile = dest.with_suffix('.json')
-        log.debug('Saving config as %s', configfile)
-        with configfile.open('w', encoding='utf-8') as f:
-            json.dump(config, f)
-        cmd = [
-            'podman',
-            'run',
-            '--rm',
-            '-it',
-            '-v',
-            f'{tmpdir}:/import:ro,z',
-            '--env-host',
-            'docker.io/fireflyiii/data-importer',
-        ]
-        if log.isEnabledFor(logging.DEBUG):
-            log.debug(
-                'Running command: %s',
-                ' '.join(shlex.quote(str(a)) for a in cmd),
-            )
-        if os.environ.get('DEBUG_SKIP_IMPORT'):
-            cmd = ['true']
-        p = subprocess.run(cmd, env=env, check=False)
-        if p.returncode == 0:
-            log.info(
-                'Successfully imported transactions from %s to Firefly III',
-                csv,
-            )
-        else:
-            log.error('Failed to import transactions from %s')
 
 
 def get_last_transaction_date(key: int, token: str) -> datetime.date:
@@ -201,12 +200,13 @@ def get_last_transaction_date(key: int, token: str) -> datetime.date:
                 continue
             if date > last_date:
                 last_date = date
-    return last_date.date() + datetime.timedelta(days=1)
+    return last_date.date()
 
 
-def download_chase(page: Page, end_date: datetime.date, token: str) -> bool:
+def download_chase(
+    page: Page, end_date: datetime.date, token: str, importer: FireflyImporter
+) -> bool:
     with Chase(page) as c, ntfyerror('Chase', page) as r:
-        c.login()
         key = ACCOUNTS['chase']
         try:
             start_date = get_last_transaction_date(key, token)
@@ -216,23 +216,28 @@ def download_chase(page: Page, end_date: datetime.date, token: str) -> bool:
                 e,
             )
             return False
-        if start_date >= end_date:
+        if start_date > end_date:
             log.info(
                 'Skipping Chase account: last transaction was %s',
                 start_date,
             )
             return True
+        c.login()
         csv = c.download_transactions(start_date, end_date)
         log.info('Importing transactions from Chase into Firefly III')
-        c.firefly_import(csv, key, token)
+        c.firefly_import(csv, key, importer)
     return r.success
 
 
-def download_commerce(page: Page, end_date: datetime.date, token: str) -> bool:
+def download_commerce(
+    page: Page,
+    end_date: datetime.date,
+    token: str,
+    importer: FireflyImporter,
+) -> bool:
     log.info('Downloading transaction lists from Commerce Bank')
     csvs = []
     with CommerceBank(page) as c, ntfyerror('Commerce Bank', page) as r:
-        c.login()
         for name, key in ACCOUNTS['commerce'].items():
             try:
                 start_date = get_last_transaction_date(key, token)
@@ -243,7 +248,7 @@ def download_commerce(page: Page, end_date: datetime.date, token: str) -> bool:
                     e,
                 )
                 continue
-            if start_date >= end_date:
+            if start_date > end_date:
                 log.info(
                     'Skipping account %s: last transaction was %s',
                     name,
@@ -255,11 +260,12 @@ def download_commerce(page: Page, end_date: datetime.date, token: str) -> bool:
                 start_date,
                 name,
             )
+            c.login()
             c.open_account(name)
             csvs.append((key, c.download_transactions(start_date, end_date)))
         log.info('Importing transactions from Commerce Bank into Firefly III')
         for key, csv in csvs:
-            c.firefly_import(csv, key, token)
+            c.firefly_import(csv, key, importer)
     return r.success
 
 
@@ -283,6 +289,8 @@ class ntfyerror:
             log.exception(
                 'Swallowed exception:', exc_info=(exc_type, exc_value, tb)
             )
+            if os.environ.get('DEBUG_NTFY', '1') == '0':
+                return True
             if ss := self.page.screenshot():
                 save_screenshot(ss)
             ntfy(
@@ -377,6 +385,8 @@ class CommerceBank:
         self.logout()
 
     def login(self) -> None:
+        if self._logged_in:
+            return
         log.debug('Navigating to %s', self.URL)
         self.page.goto(self.URL)
         password = rbw_get(self.vault_item, self.vault_folder, self.username)
@@ -451,10 +461,12 @@ class CommerceBank:
         modal.get_by_label('Close').click()
         return path
 
-    def firefly_import(self, csv: Path, account: int, token: str) -> None:
+    def firefly_import(
+        self, csv: Path, account: int, importer: FireflyImporter
+    ) -> None:
         config = copy.deepcopy(self.IMPORT_CONFIG)
         config['default_account'] = account
-        firefly_import(csv, config, token)
+        importer.import_csv(csv, config)
 
 
 class Chase:
@@ -560,6 +572,8 @@ class Chase:
             log.info('Successfully saved cookies to %s', self.saved_cookies)
 
     def login(self) -> None:
+        if self._logged_in:
+            return
         log.debug('Navigating to %s', self.URL)
         self.page.goto(self.URL)
         self.page.wait_for_load_state()
@@ -635,7 +649,9 @@ class Chase:
             self.page.get_by_role('button', name='Sign out').click()
         log.info('Logged out of Chase')
 
-    def firefly_import(self, csv: Path, account: int, token: str) -> None:
+    def firefly_import(
+        self, csv: Path, account: int, importer: FireflyImporter
+    ) -> None:
         config = copy.deepcopy(self.IMPORT_CONFIG)
         config['default_account'] = account
         with csv.open('r', encoding='utf-8') as f:
@@ -648,27 +664,32 @@ class Chase:
             config['do_mapping'].pop(0)
         else:
             raise ValueError(f'Unexpected CSV schema: {headers}')
-        firefly_import(csv, config, token)
+        importer.import_csv(csv, config)
 
 
 def main() -> None:
     logging.basicConfig(level=logging.DEBUG)
-    if not rbw_unlocked():
-        ntfy(
-            'xactfetch needs you to unlock the rbw vault',
-            tags='closed_lock_with_key',
-        )
     log.debug('Getting Firefly III access token from rbw vault')
     token = rbw_get('xactfetch')
+    import_secret = secret_from_file(
+        'FIREFLY_IMPORT_SECRET_FILE', 'import.secret'
+    )
+    import_auth = (
+        os.environ.get('FIREFLY_IMPORT_USER', getpass.getuser()),
+        secret_from_file('FIREFLY_IMPORT_PASSWORD_FILE', 'import.password'),
+    )
+    importer = FireflyImporter(
+        FIREFLY_III_IMPORTER_URL, import_secret, import_auth
+    )
     end_date = datetime.date.today() - datetime.timedelta(days=1)
     with sync_playwright() as pw:
         headless = os.environ.get('DEBUG_HEADLESS_BROWSER', '1') == '1'
         browser = pw.firefox.launch(headless=headless)
         page = browser.new_page()
         failed = False
-        if not download_commerce(page, end_date, token):
+        if not download_commerce(page, end_date, token, importer):
-           failed = True
+            failed = True
-        if not download_chase(page, end_date, token):
+        if not download_chase(page, end_date, token, importer):
             failed = True
     raise SystemExit(1 if failed else 0)
 

xactfetch.timer

@@ -0,0 +1,10 @@
+[Unit]
+Description=Daily bank transaction fetch
+
+[Timer]
+OnCalendar=9:00
+RandomizedDelaySec=2h
+AccuracySec=10m
+
+[Install]
+WantedBy=default.target