Add support for simple web server

This commit is contained in:
2025-04-04 15:40:50 +02:00
parent d5d1218f5c
commit 900145a4d0
9 changed files with 1410 additions and 197 deletions

View File

@@ -5,7 +5,7 @@ self: {
system,
...
}: let
cfg = config.programs.actual-importer;
cfg = config.services.actual-importer;
appConfig = (pkgs.formats.yaml {}).generate "actual-importer.config.yaml" cfg.config;
@@ -13,14 +13,32 @@ self: {
name = "actual-importer";
runtimeInputs = [self.packages.${system}.default];
text = ''
actual-importer -c "${appConfig}" "$@";
actual-importer "$@" -c "${appConfig}";
'';
};
in
with lib; {
options.programs.actual-importer = {
options.services.actual-importer = {
enable = mkEnableOption "actual-importer";
server = {
enable = mkEnableOption "actual-importer server";
port = mkOption {
type = types.port;
description = "The port on which importer will be listening on";
default = 3000;
example = 8080;
};
openFirewall = mkOption {
type = types.bool;
description = "Whether the configured port should be opened on firewall";
default = false;
example = true;
};
};
config = mkOption {
type = types.attrs;
description = "The actual-importer config which will be eventually converted to yaml";
@@ -49,5 +67,21 @@ in
config = mkIf cfg.enable {
environment.systemPackages = [app];
systemd.services.actual-importer-server = mkIf cfg.server.enable {
enable = true;
description = "ActualBudget importer server";
wants = ["network.target"];
after = ["network.target"];
wantedBy = ["multi-user.target"];
serviceConfig = {
ExecStart = "${self.packages.${system}.default}/bin/actual-importer serve ${toString cfg.server.port} -c ${appConfig}";
};
};
networking.firewall.allowedTCPPorts = mkIf cfg.server.openFirewall [cfg.server.port];
};
}

1032
package-lock.json generated

File diff suppressed because it is too large Load Diff

View File

@@ -20,9 +20,13 @@
},
"dependencies": {
"@actual-app/api": "^25.3.1",
"@types/express": "^5.0.1",
"@types/multer": "^1.4.12",
"@types/papaparse": "^5.3.15",
"commander": "^13.1.0",
"express": "^5.1.0",
"iconv-lite": "^0.6.3",
"multer": "^1.4.5-lts.2",
"papaparse": "^5.5.2",
"yaml": "^2.7.1"
}

View File

@@ -7,5 +7,5 @@ buildNpmPackage {
pname = "actual-importer";
version = "0.0.1";
src = ./.;
npmDepsHash = "sha256-u880X9C5s69nFU0uMt6pRLRGgCui0Pwx1K9lrOroPYw=";
npmDepsHash = "sha256-dbMJNdYEZvvUTLHTN9W9VgDwrODdO1eZpfIwxlIvimA=";
}

139
src/backend/index.ts Normal file
View File

@@ -0,0 +1,139 @@
const { mkdir } = require('node:fs/promises');
import * as api from "@actual-app/api";
import { ServerConfig } from "@/types/config";
import { Transaction } from "@/types/transaction";
import { enhancedStringConfig } from "@/util/config";
import { utils } from "@actual-app/api";
export type ActualImportResult = Awaited<ReturnType<typeof api.importTransactions>>;
type ActualTransaction = {
id?: string;
account?: string;
date: string;
amount?: number;
payee?: string;
payee_name?: string;
imported_payee?: string;
category_id?: string;
notes?: string;
imported_id?: string;
transfer_id?: string;
cleared?: boolean;
subtransactions?: ActualTransaction[];
};
type ActualAccount = {
id: string;
name: string;
type: string;
offBudget: boolean;
closed: boolean;
};
type ActualPayee = {
id: string;
name: string;
category?: string;
transfer_acct?: string;
};
export class Actual {
#config: ServerConfig;
#dryRun: boolean;
constructor(config: ServerConfig, dryRun: boolean = false) {
this.#config = config;
this.#dryRun = dryRun;
}
#map(transaction: Transaction, accounts: ActualAccount[], payees: ActualPayee[]): ActualTransaction {
const actualTransaction: ActualTransaction = {
imported_id: transaction.id,
date: transaction.date,
amount: utils.amountToInteger(transaction.amount),
notes: transaction.title,
imported_payee: transaction.to,
};
const findAccount = (name: string) => {
const accountName = this.#config.accountAliases?.[name] ?? name;
const account = accounts.find(a => a.name === accountName);
if (!account) {
throw new Error(`Unknown account: ${accountName}`);
}
return account;
}
const findTransferPayee = (name: string) => {
const account = findAccount(name);
const payee = payees.find(p => p.transfer_acct === account.id);
if (!payee) {
throw new Error(`Transfer payee not found for account ${name} with ID ${account.id}`);
}
return payee;
}
switch(transaction.kind) {
case 'regular':
actualTransaction.account = findAccount(transaction.from).id;
actualTransaction.payee_name = transaction.to;
break;
case 'transfer':
actualTransaction.account = findAccount(transaction.to).id;
actualTransaction.payee = findTransferPayee(transaction.from).id;
break;
}
return actualTransaction;
}
async submit(transactions: Transaction[]): Promise<ActualImportResult> {
try {
await mkdir(this.#config.data, { recursive: true });
} catch(e) {}
await api.init({
serverURL: this.#config.url,
password: enhancedStringConfig(this.#config.password),
dataDir: this.#config.data
});
await api.downloadBudget(this.#config.budget);
const accounts: ActualAccount[] = await api.getAccounts();
const payees: ActualPayee[] = await api.getPayees();
const toImport = transactions.map(t => this.#map(t, accounts, payees))
const output: ActualImportResult = { added: [], updated: [], errors: [] };
if (this.#dryRun) for (const transaction of toImport ){
console.log(`${transaction.imported_payee} ::: ${transaction.notes} ::: ${transaction.amount}`);
}
else for (const transaction of toImport) {
const result = await api.importTransactions(transaction.account, [transaction]);
output.added.push(...result.added);
output.updated.push(...result.updated);
output.errors?.push(...(result.errors ?? []));
await sleep(100);
}
await api.shutdown();
return output;
}
}
async function sleep(ms: number): Promise<void> {
return new Promise(resolve => setTimeout(resolve, ms));
}

View File

@@ -1,26 +1,61 @@
import fs from "fs";
import { program } from "commander";
import { CLIOptions } from "@/types/cli";
import { ImportOptions } from "@/types/cli";
import { loadConfig } from "./config";
import { loadTransactions } from "@/runner";
import { importTransactions } from "@/runner";
import { serve } from "@/server";
export function run(...args: string[]) {
program
.name("actual-importer")
.version("0.0.1")
program
.command("import <csv-file>")
.requiredOption("-c, --config <file>", "sets the path to the YAML file with configuration")
.option("-d, --dry-run", "simulates the import by printing out what will be imported")
.option("-p, --profile <name>", "sets the desired profile to invoke")
.option("-s, --server <name>", "sets the desired server to upload transactions to")
.option("-x, --set <arg>", "overrides the config option for this specific run (arg: <key>=<name>, i.e. profiles.myprofile.parser=pl.ing", (v: string, prev: string[]) => prev.concat([v]), [])
.argument('<csv-file>', 'CSV file to be imported')
.action(handle)
.parse(args);
.action(doImport)
program
.command("serve <port>")
.requiredOption("-c, --config <file>", "sets the path to the YAML file with configuration")
.option("-x, --set <arg>", "overrides the config option for this specific run (arg: <key>=<name>, i.e. profiles.myprofile.parser=pl.ing", (v: string, prev: string[]) => prev.concat([v]), [])
.action(doServe)
program.parse(args);
}
function handle(file: string, options: CLIOptions) {
const config = loadConfig(options.config);
async function doImport(file: string, options: ImportOptions) {
const config = parseConfig(options.config, options.set);
for (const override of options.set) {
if (Object.keys(config?.profiles ?? {}).length === 0) {
throw new Error(`No profiles defined in the ${options.config}`);
}
if (Object.keys(config?.servers ?? {}).length === 0) {
throw new Error(`No servers defined in the ${options.config}`);
}
const profile = options.profile ?? config.defaultProfile ?? Object.keys(config.profiles)[0];
const server = options.server ?? config.defaultServer ?? Object.keys(config.servers)[0];
const result = await importTransactions(fs.createReadStream(file), profile, server, config, options.dryRun);
}
function doServe(port: string, options: any) {
const config = parseConfig(options.config, options.set);
serve(config, Number.parseInt(port));
}
function parseConfig(path: string, set: string[]) {
const config = loadConfig(path);
for (const override of set) {
const [path, value] = override.split("=");
const segments = path.trim().split(".")
@@ -39,16 +74,5 @@ function handle(file: string, options: CLIOptions) {
});
}
if (Object.keys(config?.profiles ?? {}).length === 0) {
throw new Error(`No profiles defined in the ${options.config}`);
}
if (Object.keys(config?.servers ?? {}).length === 0) {
throw new Error(`No servers defined in the ${options.config}`);
}
const profile = options.profile ?? config.defaultProfile ?? Object.keys(config.profiles)[0];
const server = options.server ?? config.defaultServer ?? Object.keys(config.servers)[0];
loadTransactions(file, profile, server, config, options.dryRun);
}
return config;
}

View File

@@ -1,55 +1,64 @@
import Papa from "papaparse";
import fs from "fs";
import iconv from "iconv-lite";
import { createParser } from "@/parser";
import { Actual } from "@/server";
import { Actual, ActualImportResult } from "@/backend";
import { Config } from "@/types/config";
import { Readable } from "stream";
import { Transaction } from "@/types/transaction";
export type ImportResult = {
transactions: Transaction[];
result: ActualImportResult;
skipped: string[][];
};
export function loadTransactions(file: string, profile: string, server: string, config: Config, dryRun?: boolean) {
const profileConfig = config.profiles[profile];
if (!profileConfig) {
throw new Error(`Unknown profile: ${profile}`);
}
export async function importTransactions(stream: Readable, profile: string, server: string, config: Config, dryRun?: boolean): Promise<ImportResult> {
return new Promise((resolve, reject) => {
const profileConfig = config.profiles[profile];
const serverConfig = config.servers[server];
if(!serverConfig) {
throw new Error(`Unknown server: ${server}`);
}
if (!profileConfig) {
throw new Error(`Unknown profile: ${profile}`);
}
const parser = createParser(profileConfig, serverConfig);
const serverConfig = config.servers[server];
if(!serverConfig) {
throw new Error(`Unknown server: ${server}`);
}
const actualServer = new Actual(serverConfig, dryRun);
const skipped: string[] = [];
const parser = createParser(profileConfig, serverConfig);
const handleRow = async (data: string[]) => {
const pushed = await parser.pushTransaction(data);
const actualServer = new Actual(serverConfig, dryRun);
const skipped: string[][] = [];
const handleRow = async (data: string[]) => {
const pushed = await parser.pushTransaction(data);
if (!pushed) {
skipped.push(data);
}
};
const handleClose = async () => {
try {
const transactions = await parser.reconcile();
const result = await actualServer.submit(transactions);
resolve({
transactions,
result,
skipped
});
} catch (e) {
console.error(e);
reject(e);
}
};
if (!pushed) {
skipped.push(`Skipped ==> ${data.join(" ::: ")}`);
}
};
const handleClose = async () => {
try {
const transactions = await parser.reconcile();
const result = await actualServer.submit(transactions);
const now = new Date();
const date = `${now.getFullYear()}-${(now.getMonth()+1).toString().padStart(2, '0')}-${now.getDate().toString().padStart(2, '0')}`;
const time = `${now.getHours().toString().padStart(2, '0')}-${now.getMinutes().toString().padStart(2, '0')}-${now.getSeconds().toString().padStart(2, '0')}`
const filename = `${serverConfig.data}/import.${profile}.${server}.${date}T${time}.json`.replaceAll(/\/+/g, "/");
const logContent = `${JSON.stringify(result, undefined, 2)}\n\n\n\n${skipped.join("\n")}`
fs.writeFileSync(filename, logContent);
console.log(`Detailed output written to ${filename} file.`);
} catch (e) {
console.error(e);
}
};
fs.createReadStream(file)
.pipe(iconv.decodeStream(profileConfig.encoding ?? "utf8"))
.pipe(Papa.parse(Papa.NODE_STREAM_INPUT))
.on('data', handleRow)
.on('close', handleClose);
}
stream
.pipe(iconv.decodeStream(profileConfig.encoding ?? "utf8"))
.pipe(Papa.parse(Papa.NODE_STREAM_INPUT))
.on('data', handleRow)
.on('close', handleClose);
});
};

View File

@@ -1,137 +1,103 @@
const { mkdir } = require('node:fs/promises');
import * as api from "@actual-app/api";
import { ServerConfig } from "@/types/config";
import { Transaction } from "@/types/transaction";
import { enhancedStringConfig } from "@/util/config";
import { utils } from "@actual-app/api";
import { ImportResult, importTransactions } from "@/runner";
import { Config } from "@/types/config";
import express from "express";
import multer from "multer";
import { Readable } from "stream";
type ActualTransaction = {
id?: string;
account?: string;
date: string;
amount?: number;
payee?: string;
payee_name?: string;
imported_payee?: string;
category_id?: string;
notes?: string;
imported_id?: string;
transfer_id?: string;
cleared?: boolean;
subtransactions?: ActualTransaction[];
};
export function serve(config: Config, port: number) {
const app = express();
const upload = multer({ storage: multer.memoryStorage() });
type ActualAccount = {
id: string;
name: string;
type: string;
offBudget: boolean;
closed: boolean;
};
app.get("/", (req, res) => {
res.send(`
<html>
<body>
<h2>Import transactions</h2>
<form action="/import" method="POST" enctype="multipart/form-data">
<p>
<label for="file">CSV File</label>
<input id="file" type="file" name="file" accept=".csv" required />
</p>
<p>
<label for="profile">Profile</label>
<select id="profile" name="profile">
${Object.keys(config.profiles).map(p => `<option${p === config.defaultProfile ? " selected": ""}>${p}</option>`)}
</select>
</p>
<p>
<label for="server">Server</label>
<select id="server" name="server">
${Object.keys(config.servers).map(s => `<option${s === config.defaultServer ? " selected": ""}>${s}</option>`)}
</select>
</p>
<p>
<button type="submit">Import</button>
</p>
</form>
</body>
</html>
`);
});
type ActualPayee = {
id: string;
name: string;
category?: string;
transfer_acct?: string;
};
export class Actual {
#config: ServerConfig;
#dryRun: boolean;
constructor(config: ServerConfig, dryRun: boolean = false) {
this.#config = config;
this.#dryRun = dryRun;
}
#map(transaction: Transaction, accounts: ActualAccount[], payees: ActualPayee[]): ActualTransaction {
const actualTransaction: ActualTransaction = {
imported_id: transaction.id,
date: transaction.date,
amount: utils.amountToInteger(transaction.amount),
notes: transaction.title,
imported_payee: transaction.to,
};
const findAccount = (name: string) => {
const accountName = this.#config.accountAliases?.[name] ?? name;
const account = accounts.find(a => a.name === accountName);
if (!account) {
throw new Error(`Unknown account: ${accountName}`);
}
return account;
app.post("/import", upload.single('file'), async (req, res) => {
if (!req.file) {
throw new Error("No file to upload");
}
const findTransferPayee = (name: string) => {
const account = findAccount(name);
const payee = payees.find(p => p.transfer_acct === account.id);
const { profile, server } = req.body;
if (!payee) {
throw new Error(`Transfer payee not found for account ${name} with ID ${account.id}`);
}
const stream = new Readable();
stream.push(req.file.buffer);
stream.push(null);
return payee;
}
const result = await importTransactions(stream, profile, server, config);
res.send(formatResult(result));
});
switch(transaction.kind) {
case 'regular':
actualTransaction.account = findAccount(transaction.from).id;
actualTransaction.payee_name = transaction.to;
break;
case 'transfer':
actualTransaction.account = findAccount(transaction.to).id;
actualTransaction.payee = findTransferPayee(transaction.from).id;
break;
}
return actualTransaction;
}
async submit(transactions: Transaction[]): ReturnType<typeof api.importTransactions> {
try {
await mkdir(this.#config.data, { recursive: true });
} catch(e) {}
await api.init({
serverURL: this.#config.url,
password: enhancedStringConfig(this.#config.password),
dataDir: this.#config.data
});
await api.downloadBudget(this.#config.budget);
const accounts: ActualAccount[] = await api.getAccounts();
const payees: ActualPayee[] = await api.getPayees();
const toImport = transactions.map(t => this.#map(t, accounts, payees))
const output: Awaited<ReturnType<typeof api.importTransactions>> = { added: [], updated: [], errors: [] };
if (this.#dryRun) for (const transaction of toImport ){
console.log(`${transaction.imported_payee} ::: ${transaction.notes} ::: ${transaction.amount}`);
}
else for (const transaction of toImport) {
const result = await api.importTransactions(transaction.account, [transaction]);
output.added.push(...result.added);
output.updated.push(...result.updated);
output.errors?.push(...(result.errors ?? []));
await sleep(100);
}
await api.shutdown();
return output;
}
app.listen(port, () => {
console.log(`Server running on ${port} port`);
});
}
async function sleep(ms: number): Promise<void> {
return new Promise(resolve => setTimeout(resolve, ms));
function formatResult(result: ImportResult): string {
return `
<html>
<body>
<h2>Import Result</h2>
<h3>Considered transactions (${result.transactions.length})</h3>
<table>
<thead>
<tr>
<th>ID</th>
<th>Kind</th>
<th>Date</th>
<th>From</th>
<th>To</th>
<th>Amount</th>
<th>Title</th>
</th>
</thead>
<tbody>
${result.transactions.map(t => `<tr><td>${t.id}</td><td>${t.kind}</td><td>${t.date}</td><td>${t.from}</td><td>${t.to}</td><td>${t.amount}</td><td>${t.title}</td></tr>`)}
</tbody>
</table>
<h3>Import status</h3>
<p>
<ul>
<li>Added: <strong>${result.result.added.length}</strong></li>
<li>Updated: <strong>${result.result.updated.length}</strong></li>
<li>Errors: <strong>${result.result?.errors?.length ?? 0}</strong></li>
</ul>
</p>
${(result.result.errors?.length ?? 0) > 0
? `<p>Errors: <ul>${result.result.errors?.map(e => `<li>${e.message}</li>`)}</ul></p>`
: ""}
<h3>Skipped CSV rows</h3>
<p>The <tt>:::</tt> is CSV field delimiter</p>
<pre>${result.skipped.map(d => d.join(" ::: ")).join("\n")}</pre>
</body>
</html>
`;
}

View File

@@ -1,7 +1,12 @@
export type CLIOptions = {
export type ImportOptions = {
config: string;
dryRun?: boolean;
profile?: string;
server?: string;
server?: string;
set: string[];
};
export type ServeOptions = {
config: string;
set: string[];
};