bartek/actual-importer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Split preparation and submission of transaction in Actual backend

Browse Source
This commit is contained in:
Bartłomiej Pluta
2025-05-06 14:12:54 +02:00
parent 47892f8262
commit 23b92aa90d
5 changed files with 147 additions and 130 deletions
Download Patch File Download Diff File Expand all files Collapse all files

114
src/backend/index.ts
View File

@@ -7,6 +7,11 @@ import { utils } from "@actual-app/api";
export type ActualImportResult = Awaited<ReturnType<typeof api.importTransactions>>; export type ActualImportResult = Awaited<ReturnType<typeof api.importTransactions>>;
export type SubmitOptions =
| { mode: 'add', learnCategories?: boolean, runTransfers?: boolean }
| { mode: 'import' }
| { mode: 'dry-run' };
type ActualTransaction = { type ActualTransaction = {
id?: string; id?: string;
account?: string; account?: string;
@@ -40,11 +45,9 @@ type ActualPayee = {
export class Actual { export class Actual {
#config: ServerConfig; #config: ServerConfig;
#dryRun: boolean;
constructor(config: ServerConfig, dryRun: boolean = false) { constructor(config: ServerConfig) {
this.#config = config; this.#config = config;
this.#dryRun = dryRun;
} }
#map(transaction: Transaction, accounts: ActualAccount[], payees: ActualPayee[]): ActualTransaction { #map(transaction: Transaction, accounts: ActualAccount[], payees: ActualPayee[]): ActualTransaction {
@@ -94,21 +97,7 @@ export class Actual {
return actualTransaction; return actualTransaction;
} }
async import(transactions: Transaction[]): Promise<ActualImportResult> { async #api<T>(fn: () => Promise<T>): Promise<T> {
return this.#submit(transactions, t => api.importTransactions(t.account, [t]));
}
async add(transactions: Transaction[], learnCategories: boolean, runTransfers: boolean): Promise<ActualImportResult> {
return this.#submit(transactions, async t => {
const result = await api.addTransactions(t.account, [t], { learnCategories, runTransfers });
return result === "ok"
? { added: [t], updated: [], errors: [] }
: { added: [], updated: [], errors: [{ message: `Cannot add transaction: ${t}` }] };
});
}
async #submit(transactions: Transaction[], add: (t: ActualTransaction) => Promise<ActualImportResult>): Promise<ActualImportResult> {
try { try {
await mkdir(this.#config.data, { recursive: true }); await mkdir(this.#config.data, { recursive: true });
} catch(e) {} } catch(e) {}
@@ -120,33 +109,34 @@ export class Actual {
}); });
await api.downloadBudget(this.#config.budget); await api.downloadBudget(this.#config.budget);
const accounts: ActualAccount[] = await api.getAccounts();
const payees: ActualPayee[] = await api.getPayees();
const toImport = transactions.map(t => this.#map(t, accounts, payees)) try {
return await fn();
} finally {
await api.shutdown();
};
}
async #printTransactions(transactions: ActualTransaction[]) {
const data: { value: (t: ActualTransaction) => string|undefined, header: string, pad: number }[] = [
{ value: t => t.payee !== undefined ? "Yes" : "No", header: "Known payee", pad: 12 },
{ value: t => t.imported_payee, header: "Imported payee", pad: 70 },
{ value: t => t.amount?.toString(), header: "Amount", pad: 10 },
{ value: t => t.notes, header: "Notes", pad: 100 }
];
const header = data.map(d => d.header.padEnd(d.pad)).join(" ");
const rows = transactions.map(transaction => data.map(d => (d.value(transaction) ?? "").padEnd(d.pad)).join(" "));
console.log(header);
console.log("-".repeat(Math.max(...rows.map(x => x.length)) ?? header.length));
rows.forEach(x => console.log(x));
}
async #submitTransactions(transactions: ActualTransaction[], add: (t: ActualTransaction) => Promise<ActualImportResult>): Promise<ActualImportResult> {
const output: ActualImportResult = { added: [], updated: [], errors: [] }; const output: ActualImportResult = { added: [], updated: [], errors: [] };
if (this.#dryRun) { for (const transaction of transactions) {
const data: { value: (t: ActualTransaction) => string|undefined, header: string, pad: number }[] = [
{ value: t => t.payee !== undefined ? "Yes" : "No", header: "Known payee", pad: 12 },
{ value: t => t.imported_payee, header: "Imported payee", pad: 70 },
{ value: t => t.amount?.toString(), header: "Amount", pad: 10 },
{ value: t => t.notes, header: "Notes", pad: 100 }
];
const header = data.map(d => d.header.padEnd(d.pad)).join(" ");
const rows = toImport.map(transaction => data.map(d => (d.value(transaction) ?? "").padEnd(d.pad)).join(" "));
console.log(header);
console.log("-".repeat(Math.max(...rows.map(x => x.length)) ?? header.length));
rows.forEach(x => console.log(x));
}
else for (const transaction of toImport) {
const result = await add(transaction); const result = await add(transaction);
output.added.push(...result.added); output.added.push(...result.added);
@@ -155,11 +145,47 @@ export class Actual {
await sleep(100); await sleep(100);
} }
await api.shutdown();
return output; return output;
} }
async #doSubmit(transactions: ActualTransaction[], opts: SubmitOptions): Promise<ActualImportResult> {
switch(opts.mode) {
case 'add':
return this.#submitTransactions(transactions, async t => {
const result = await api.addTransactions(t.account, [t], opts);
return result === "ok"
? { added: [t], updated: [], errors: [] }
: { added: [], updated: [], errors: [{ message: `Cannot add transaction: ${t}` }] };
});
case 'import':
return this.#submitTransactions(transactions, t => api.importTransactions(t.account, [t]));
}
this.#printTransactions(transactions);
return { added: [], updated: [], errors: [] };
}
async prepare(transactions: Transaction[]): Promise<ActualTransaction[]> {
const accounts: ActualAccount[] = await api.getAccounts();
const payees: ActualPayee[] = await api.getPayees();
return transactions.map(t => this.#map(t, accounts, payees));
}
async submit(transactions: ActualTransaction[], opts: SubmitOptions): Promise<ActualImportResult> {
return this.#api(() => this.#doSubmit(transactions, opts));
}
async load(transactions: Transaction[], opts: SubmitOptions): Promise<ActualImportResult> {
return this.#api(async () => {
const prepared = await this.prepare(transactions);
return this.#doSubmit(prepared, opts);
});
}
} }
async function sleep(ms: number): Promise<void> { async function sleep(ms: number): Promise<void> {

132
src/cli/index.ts
View File

@@ -2,81 +2,29 @@ import fs from "fs";
import { program } from "commander"; import { program } from "commander";
import { AddOptions, ImportOptions } from "@/types/cli"; import { AddOptions, ImportOptions } from "@/types/cli";
import { loadConfig } from "./config"; import { loadConfig } from "./config";
import { addTransactions, importTransactions } from "@/runner"; import { submitTransactions } from "@/runner";
import { serve } from "@/server"; import { serve } from "@/server";
import { SubmitOptions } from "@/backend";
export function run(...args: string[]) { function doSubmit<O extends ImportOptions | AddOptions>(parseOpts: (o: O) => SubmitOptions) {
program return async (file: string, options: O) => {
.name("actual-importer") const config = parseConfig(options.config, options.set);
.version("0.0.1")
if (Object.keys(config?.profiles ?? {}).length === 0) {
throw new Error(`No profiles defined in the ${options.config}`);
}
if (Object.keys(config?.servers ?? {}).length === 0) {
throw new Error(`No servers defined in the ${options.config}`);
}
const profile = options.profile ?? config.defaultProfile ?? Object.keys(config.profiles)[0];
const server = options.server ?? config.defaultServer ?? Object.keys(config.servers)[0];
program const opts: SubmitOptions = options.dryRun ? { mode: 'dry-run' } : parseOpts(options);
.command("add <csv-file>")
.requiredOption("-c, --config <file>", "sets the path to the YAML file with configuration") await submitTransactions(fs.createReadStream(file), profile, server, config, opts);
.option("-d, --dry-run", "simulates the import by printing out what will be imported")
.option("-p, --profile <name>", "sets the desired profile to invoke")
.option("-s, --server <name>", "sets the desired server to upload transactions to")
.option("-t, --transfers", "whether transfer transactions should be resolved")
.option("-l, --learn", "whether new rules for category assignment should be created")
.option("-x, --set <arg>", "overrides the config option for this specific run (arg: <key>=<name>, i.e. profiles.myprofile.parser=pl.ing", (v: string, prev: string[]) => prev.concat([v]), [])
.action(doAdd)
program
.command("import <csv-file>")
.requiredOption("-c, --config <file>", "sets the path to the YAML file with configuration")
.option("-d, --dry-run", "simulates the import by printing out what will be imported")
.option("-p, --profile <name>", "sets the desired profile to invoke")
.option("-s, --server <name>", "sets the desired server to upload transactions to")
.option("-x, --set <arg>", "overrides the config option for this specific run (arg: <key>=<name>, i.e. profiles.myprofile.parser=pl.ing", (v: string, prev: string[]) => prev.concat([v]), [])
.action(doImport)
program
.command("serve <port>")
.requiredOption("-c, --config <file>", "sets the path to the YAML file with configuration")
.option("-x, --set <arg>", "overrides the config option for this specific run (arg: <key>=<name>, i.e. profiles.myprofile.parser=pl.ing", (v: string, prev: string[]) => prev.concat([v]), [])
.action(doServe)
program.parse(args);
}
async function doAdd(file: string, options: AddOptions) {
const config = parseConfig(options.config, options.set);
if (Object.keys(config?.profiles ?? {}).length === 0) {
throw new Error(`No profiles defined in the ${options.config}`);
} }
if (Object.keys(config?.servers ?? {}).length === 0) {
throw new Error(`No servers defined in the ${options.config}`);
}
const profile = options.profile ?? config.defaultProfile ?? Object.keys(config.profiles)[0];
const server = options.server ?? config.defaultServer ?? Object.keys(config.servers)[0];
const result = await addTransactions(fs.createReadStream(file), profile, server, config, options.dryRun, options.learn, options.transfers);
}
async function doImport(file: string, options: ImportOptions) {
const config = parseConfig(options.config, options.set);
if (Object.keys(config?.profiles ?? {}).length === 0) {
throw new Error(`No profiles defined in the ${options.config}`);
}
if (Object.keys(config?.servers ?? {}).length === 0) {
throw new Error(`No servers defined in the ${options.config}`);
}
const profile = options.profile ?? config.defaultProfile ?? Object.keys(config.profiles)[0];
const server = options.server ?? config.defaultServer ?? Object.keys(config.servers)[0];
const result = await importTransactions(fs.createReadStream(file), profile, server, config, options.dryRun);
}
function doServe(port: string, options: any) {
const config = parseConfig(options.config, options.set);
serve(config, Number.parseInt(port));
} }
function parseConfig(path: string, set: string[]) { function parseConfig(path: string, set: string[]) {
@@ -103,3 +51,45 @@ function parseConfig(path: string, set: string[]) {
return config; return config;
} }
export function run(...args: string[]) {
program
.name("actual-importer")
.version("0.0.1")
program
.command("add <csv-file>")
.requiredOption("-c, --config <file>", "sets the path to the YAML file with configuration")
.option("-d, --dry-run", "simulates the import by printing out what will be imported")
.option("-p, --profile <name>", "sets the desired profile to invoke")
.option("-s, --server <name>", "sets the desired server to upload transactions to")
.option("-t, --transfers", "whether transfer transactions should be resolved")
.option("-l, --learn", "whether new rules for category assignment should be created")
.option("-x, --set <arg>", "overrides the config option for this specific run (arg: <key>=<name>, i.e. profiles.myprofile.parser=pl.ing", (v: string, prev: string[]) => prev.concat([v]), [])
.action(doSubmit<AddOptions>(o => ({
mode: 'add',
learnCategories: o.learn,
runTransfers: o.transfers
})));
program
.command("import <csv-file>")
.requiredOption("-c, --config <file>", "sets the path to the YAML file with configuration")
.option("-d, --dry-run", "simulates the import by printing out what will be imported")
.option("-p, --profile <name>", "sets the desired profile to invoke")
.option("-s, --server <name>", "sets the desired server to upload transactions to")
.option("-x, --set <arg>", "overrides the config option for this specific run (arg: <key>=<name>, i.e. profiles.myprofile.parser=pl.ing", (v: string, prev: string[]) => prev.concat([v]), [])
.action(doSubmit<ImportOptions>(o => ({ mode: 'import' })))
program
.command("serve <port>")
.requiredOption("-c, --config <file>", "sets the path to the YAML file with configuration")
.option("-x, --set <arg>", "overrides the config option for this specific run (arg: <key>=<name>, i.e. profiles.myprofile.parser=pl.ing", (v: string, prev: string[]) => prev.concat([v]), [])
.action((port: string, options: any) => {
const config = parseConfig(options.config, options.set);
serve(config, Number.parseInt(port));
});
program.parse(args);
}

15
src/runner/index.ts
View File

@@ -1,7 +1,7 @@
import Papa from "papaparse"; import Papa from "papaparse";
import iconv from "iconv-lite"; import iconv from "iconv-lite";
import { createParser } from "@/parser"; import { createParser } from "@/parser";
import { Actual, ActualImportResult } from "@/backend"; import { Actual, ActualImportResult, SubmitOptions } from "@/backend";
import { Config } from "@/types/config"; import { Config } from "@/types/config";
import { Readable } from "stream"; import { Readable } from "stream";
import { Transaction } from "@/types/transaction"; import { Transaction } from "@/types/transaction";
@@ -12,7 +12,7 @@ export type ImportResult = {
skipped: string[][]; skipped: string[][];
}; };
const submitTransactions = (load: (server: Actual, t: Transaction[]) => Promise<ActualImportResult>) => async (stream: Readable, profile: string, server: string, config: Config, dryRun?: boolean): Promise<ImportResult> => new Promise((resolve, reject) => { export const submitTransactions = async (stream: Readable, profile: string, server: string, config: Config, opts: SubmitOptions): Promise<ImportResult> => new Promise((resolve, reject) => {
const profileConfig = config.profiles[profile]; const profileConfig = config.profiles[profile];
if (!profileConfig) { if (!profileConfig) {
@@ -26,7 +26,7 @@ const submitTransactions = (load: (server: Actual, t: Transaction[]) => Promise<
const parser = createParser(profileConfig, serverConfig); const parser = createParser(profileConfig, serverConfig);
const actualServer = new Actual(serverConfig, dryRun); const actualServer = new Actual(serverConfig);
const skipped: string[][] = []; const skipped: string[][] = [];
const handleRow = async (data: string[]) => { const handleRow = async (data: string[]) => {
@@ -40,7 +40,7 @@ const submitTransactions = (load: (server: Actual, t: Transaction[]) => Promise<
const handleClose = async () => { const handleClose = async () => {
try { try {
const transactions = await parser.reconcile(); const transactions = await parser.reconcile();
const result = await load(actualServer, transactions); const result = await actualServer.load(transactions, opts);
resolve({ resolve({
transactions, transactions,
@@ -59,9 +59,4 @@ const submitTransactions = (load: (server: Actual, t: Transaction[]) => Promise<
.pipe(Papa.parse(Papa.NODE_STREAM_INPUT, profileConfig.csv ?? parser.csvConfig)) .pipe(Papa.parse(Papa.NODE_STREAM_INPUT, profileConfig.csv ?? parser.csvConfig))
.on('data', handleRow) .on('data', handleRow)
.on('close', handleClose); .on('close', handleClose);
}); });
export const importTransactions = submitTransactions((s, t) => s.import(t));
export const addTransactions = async (stream: Readable, profile: string, server: string, config: Config, dryRun?: boolean, learnCategories?: boolean, runTransfers?: boolean): Promise<ImportResult> =>
submitTransactions((s, t) => s.add(t, !!learnCategories, !!runTransfers))(stream, profile, server, config, dryRun);

14
src/server/index.ts
View File

@@ -1,4 +1,5 @@
import { addTransactions, ImportResult, importTransactions } from "@/runner"; import { SubmitOptions } from "@/backend";
import { ImportResult, submitTransactions } from "@/runner";
import { Config } from "@/types/config"; import { Config } from "@/types/config";
import express from "express"; import express from "express";
import multer from "multer"; import multer from "multer";
@@ -65,9 +66,14 @@ export function serve(config: Config, port: number) {
stream.push(req.file.buffer); stream.push(req.file.buffer);
stream.push(null); stream.push(null);
const result = mode === "add" const opts: SubmitOptions = {
? await addTransactions(stream, profile, server, config, false, readCheckbox(learn), readCheckbox(transfers)) mode: mode as 'import'|'add',
: await importTransactions(stream, profile, server, config); learnCategories: readCheckbox(learn),
runTransfers: readCheckbox(transfers)
};
const result = await submitTransactions(stream, profile, server, config, opts);
res.send(formatResult(result)); res.send(formatResult(result));
}); });

2
src/util/parser.ts
View File

@@ -66,5 +66,5 @@ export function parseAmount(input?: string): number|undefined {
} }
export function deduplicateSpaces(input: string): string { export function deduplicateSpaces(input: string): string {
return input.replaceAll(/\s+/, " "); return input.replaceAll(/\s+/g, " ");
} }