Implement support for ING transfers

2025-04-02 21:37:57 +02:00
parent 019a3e7cd4
commit 89b59dee50
9 changed files with 312 additions and 89 deletions
--- a/src/parser/base.ts
+++ b/src/parser/base.ts
@@ -1,16 +1,19 @@
 import { Transaction } from "@/types/transaction";
-import { ProfileConfig, ParserConfig } from "@/types/config";
-import { utils } from "@actual-app/api";
+import { ProfileConfig, ParserConfig, ServerConfig } from "@/types/config";

 export abstract class BaseTransactionParser<C extends ParserConfig> {
  public readonly name: string;
-  protected abstract requiredFields: readonly (keyof C)[];
-  abstract parse(config: C, data: string[]): Promise<Transaction|undefined>;
+  protected abstract requiredFields: readonly (keyof C)[];  

  constructor(name: string) {
    this.name = name;
  }

+  public init(config: ProfileConfig, serverConfig: ServerConfig) {    
+    const cfg = config?.config as Partial<C> | undefined;   
+    this.configure({ ...config, config: this.#validate(cfg ?? {}) }, serverConfig);
+  }
+
  #validate(config: Partial<C>|undefined): C {
    for (const field of this.requiredFields) {
      if (config?.[field] === undefined) {
@@ -19,24 +22,15 @@ export abstract class BaseTransactionParser<C extends ParserConfig> {
    }

    return config as C;
+  }    
+
+  /** TO BE OVERRIDEN */
+
+  protected configure(config: Omit<ProfileConfig, 'config'> & { config: C }, serverConfig: ServerConfig) {
+
  }

-  public async parseTransaction(config: ProfileConfig, data: string[]): Promise<Transaction|undefined> {
-    const cfg = config?.config as Partial<C> | undefined;
-    return this.parse(this.#validate(cfg), data);
-  }
-
-  protected parseAmount(input?: string): number|undefined {    
-    if (input === undefined) {
-      return undefined;
-    }
-
-    const v = utils.amountToInteger(Number.parseFloat(input.replaceAll(",", ".")));
-    
-    if (isNaN(v)) {
-      return undefined;
-    }
-
-    return v;
-  }
+  abstract pushTransaction(data: string[]): Promise<boolean>; 
+  
+  abstract reconcile(): Promise<Transaction[]>;
 }
--- a/src/parser/index.ts
+++ b/src/parser/index.ts
@@ -1,8 +1,7 @@
 export { BaseTransactionParser } from "./base";
-import { ProfileConfig, ParserConfig } from "@/types/config";
+import { ProfileConfig, ParserConfig, ServerConfig } from "@/types/config";
 import { BaseTransactionParser } from "./base";
 import { default as PlIng } from "./pl/ing";
-import { Transaction } from "@/types/transaction";

 type Constructor<T extends BaseTransactionParser<ParserConfig>> = new (name: string) => T;

@@ -10,13 +9,17 @@ const PARSERS: Record<string, Constructor<BaseTransactionParser<ParserConfig>>>
  "pl.ing": PlIng
 };

-export function createParser(config: ProfileConfig): BaseTransactionParser<ParserConfig> {
+export function createParser(config: ProfileConfig, serverConfig: ServerConfig): BaseTransactionParser<ParserConfig> {
  const Parser = PARSERS[config.parser];

  if (!Parser) {
    throw new Error(`Unknown parser: ${config.parser}`);
  }

-  return new Parser(config.parser);
+  const parser = new Parser(config.parser);
+
+  parser.init(config, serverConfig);
+
+  return parser;
 }

--- a/src/parser/pl/ing/index.ts
+++ b/src/parser/pl/ing/index.ts
@@ -1,6 +1,16 @@
 import { Transaction } from "@/types/transaction";
 import { ParserConfig } from "@/types/config";
 import { BaseTransactionParser } from "../..";
+import { mapCombine, parseAmount } from "@/util/parser";
+
+type IngTransaction = {
+  [K in typeof headers[number]]: string;
+};
+
+type ValidatedIngTransaction = Omit<IngTransaction, 'transactionAmount' | 'lockAmount'> & {
+  transactionAmount?: number;
+  lockAmount?: number;
+}

 const headers = [
  'transactionDate',
@@ -26,10 +36,6 @@ const headers = [
  'unknown4',
 ];

-type IngTransaction = {
-  [K in typeof headers[number]]: string;
-};
-
 const readIngTransaction = (data: string[]): IngTransaction|undefined => {
  if (data.length !== headers.length) {
    return;
@@ -45,34 +51,101 @@ const readIngTransaction = (data: string[]): IngTransaction|undefined => {

 export default class extends BaseTransactionParser<ParserConfig> {
  protected requiredFields = [];
+  #transactions: ValidatedIngTransaction[] = [];

-  async parse(config: ParserConfig, data: string[]): Promise<Transaction | undefined> {
+  async pushTransaction(data: string[]): Promise<boolean> {
    const ing = readIngTransaction(data);
-
+    
    if (!ing) {
-      return undefined;
+      return false;
    }

    // Validate
-    if (!/^\d{4}-\d{2}-\d{2}$/.test(data[0])) {      
-      return undefined;
+    if (!/^\d{4}-\d{2}-\d{2}$/.test(data[0])) {     
+      return false;
    }

-    const amount = this.parseAmount(ing.transactionAmount);
+    const transactionAmount = parseAmount(ing.transactionAmount);    
+    const lockAmount = parseAmount(ing.lockAmount);

-    if(amount === undefined) {
-      return undefined;
+    if(transactionAmount === undefined && lockAmount === undefined) {
+      return false;
    }

-    const payee = ing.contrahentData?.trim()?.replaceAll(/\s+/g, " ");
+    this.#transactions.push({
+      ...ing,
+      transactionAmount,
+      lockAmount,
+    } as ValidatedIngTransaction);
+    return true;
+  }
+
+  async reconcile(): Promise<Transaction[]> {
+    const transactions = mapCombine(this.#transactions, "transactionNumber", this.#map, this.#combine);  
+    return transactions.reverse();
+  }
+
+  #map(transaction: ValidatedIngTransaction): Transaction {
+    const amount = transaction.transactionAmount ?? transaction.lockAmount;
+
+    if (amount === undefined) {
+      throw new Error(`Undefined amount for transaction: ${transaction}`);
+    }

    return {
+      kind: 'regular',
+      from: transaction.account,
+      to: transaction.contrahentData,
+      date: transaction.transactionDate,
+      title: transaction.title,
+      id: transaction.transactionNumber,
      amount,      
-      imported_id: ing.transactionNumber,
-      notes: ing.title,
-      date: ing.transactionDate,      
-      imported_payee: payee,
-      payee_name: payee,
+    };
+  }
+
+  #combine(a: ValidatedIngTransaction, b: ValidatedIngTransaction): Transaction {
+    const aAmount = a.transactionAmount ?? a.lockAmount;
+    const bAmount = b.transactionAmount ?? b.lockAmount;
+
+    if (a.date !== b.date) {
+      console.log(a);
+      console.log(b);
+      throw new Error(`Transfer transaction dates mismatch`);
    }
+
+    if (a.title !== b.title) {
+      console.log(a);
+      console.log(b);
+      throw new Error(`Transfer transaction titles mismatch`);
+    }
+
+    if (aAmount === undefined || bAmount === undefined) {
+      console.log(a);
+      console.log(b);
+      throw new Error(`Undefined amounts for transactions`);
+    }
+
+    const transaction: Partial<Transaction> = {
+      kind: 'transfer',
+      date: a.transactionDate,
+      title: a.title,
+      id: a.transactionNumber,
+    };
+
+    // Transaction B => A
+    if (aAmount > 0) {
+      transaction.from = b.account;
+      transaction.to = a.account;
+      transaction.amount = aAmount;      
+    }
+
+    // Transaction A => B
+    else {    
+      transaction.from = a.account;
+      transaction.to = b.account;
+      transaction.amount = bAmount;
+    }
+
+    return transaction as Transaction;
  }
 }
--- a/src/runner/index.ts
+++ b/src/runner/index.ts
@@ -17,38 +17,35 @@ export function loadTransactions(file: string, profile: string, server: string,
    throw new Error(`Unknown server: ${server}`);
  }

-  const parser = createParser(profileConfig);
+  const parser = createParser(profileConfig, serverConfig);

-  const actualServer = new Actual(serverConfig);
+  const actualServer = new Actual(serverConfig, false);
  const skipped: string[] = [];

  const handleRow = async (data: string[]) => {
-    const transaction = await parser.parseTransaction(profileConfig, data);
+    const pushed = await parser.pushTransaction(data);
    
-    if (transaction === undefined) {
+    if (!pushed) {
      skipped.push(`Skipped ==> ${data.join(" ::: ")}`);
-      return;        
    }
-
-    actualServer.pushTransaction(transaction);
  };

-  const handleClose = () => actualServer.submit()
-    .then(x => {
-      console.log(`Inserted: ${x.added.length}`);
-      console.log(`Updated: ${x.updated.length}`);
-      console.log(`Errors: ${x.errors?.length}`);
-      console.log(`Skipped: ${skipped.length}`);
+  const handleClose = async () => {
+    try {
+      const transactions = await parser.reconcile();      
+      const result = await actualServer.submit(transactions);
      const now = new Date();
      const date = `${now.getFullYear()}-${(now.getMonth()+1).toString().padStart(2, '0')}-${now.getDate().toString().padStart(2, '0')}`;
      const time = `${now.getHours().toString().padStart(2, '0')}-${now.getMinutes().toString().padStart(2, '0')}-${now.getSeconds().toString().padStart(2, '0')}`
      const filename = `${serverConfig.data}/import.${profile}.${server}.${date}T${time}.json`.replaceAll(/\/+/g, "/");

-      const logContent = `${JSON.stringify(x, undefined, 2)}\n\n\n\n${skipped.join("\n")}`
+      const logContent = `${JSON.stringify(result, undefined, 2)}\n\n\n\n${skipped.join("\n")}`
      fs.writeFileSync(filename, logContent);
      console.log(`Detailed output written to ${filename} file.`);
-    })
-    .catch(x => console.error(x))
+    } catch (e) {
+      console.error(e);
+    }
+  };
  
  fs.createReadStream(file)
    .pipe(iconv.decodeStream(profileConfig.encoding ?? "utf8"))    
--- a/src/server/index.ts
+++ b/src/server/index.ts
@@ -3,40 +3,135 @@ import * as api from "@actual-app/api";
 import { ServerConfig } from "@/types/config";
 import { Transaction } from "@/types/transaction";
 import { enhancedStringConfig } from "@/util/config";
+import { utils } from "@actual-app/api";

-type TransactionDTO = Transaction & { 
-  account: string 
+type ActualTransaction = {
+  id?: string;  
+  account?: string;
+  date: string;
+  amount?: number;
+  payee?: string;
+  payee_name?: string;
+  imported_payee?: string;
+  category_id?: string;
+  notes?: string;
+  imported_id?: string;
+  transfer_id?: string;
+  cleared?: boolean;
+  subtransactions?: ActualTransaction[];
+};
+
+type ActualAccount = {
+  id: string;
+  name: string;
+  type: string;
+  offBudget: boolean;
+  closed: boolean;
+};
+
+type ActualPayee = {
+  id: string;
+  name: string;
+  category?: string;
+  transfer_acct?: string;
 };

 export class Actual {
-  #transactions: TransactionDTO[];
  #config: ServerConfig;
+  #dryRun: boolean;

-  constructor(config: ServerConfig) {
-    this.#transactions = [];
+  constructor(config: ServerConfig, dryRun: boolean = false) {
    this.#config = config;
+    this.#dryRun = dryRun;
  }

-  pushTransaction(transaction: Transaction) {
-    this.#transactions.push({ ...transaction, account: this.#config.account });
+  #map(transaction: Transaction, accounts: ActualAccount[], payees: ActualPayee[]): ActualTransaction {
+    const actualTransaction: ActualTransaction = {
+      imported_id: transaction.id,
+      date: transaction.date,
+      amount: utils.amountToInteger(transaction.amount),
+      notes: transaction.title,
+      imported_payee: transaction.to,
+    };
+    
+    const findAccount = (name: string) => {
+      const accountName = this.#config.accountAliases?.[name] ?? name;
+      const account = accounts.find(a => a.name === accountName);
+
+      if (!account) {
+        throw new Error(`Unknown account: ${accountName}`);
+      }
+
+      return account;
+    }
+
+    const findTransferPayee = (name: string) => {
+      const account = findAccount(name);
+      const payee = payees.find(p => p.transfer_acct === account.id);
+
+      if (!payee) {
+        throw new Error(`Transfer payee not found for account ${name} with ID ${account.id}`);
+      }
+
+      return payee;
+    }
+
+    switch(transaction.kind) {
+      case 'regular':
+        actualTransaction.account = findAccount(transaction.from).id;
+        actualTransaction.payee_name = transaction.to;
+        break;
+
+      case 'transfer':
+        actualTransaction.account = findAccount(transaction.to).id;
+        actualTransaction.payee = findTransferPayee(transaction.from).id;
+        break;
+    }
+            
+    return actualTransaction;
  }

-  async submit(): ReturnType<typeof api.importTransactions> {   
+  async submit(transactions: Transaction[]): ReturnType<typeof api.importTransactions> {   
    try {
      await mkdir(this.#config.data, { recursive: true });
    } catch(e) {}
-    
+
    await api.init({ 
      serverURL: this.#config.url,
      password: enhancedStringConfig(this.#config.password),
      dataDir: this.#config.data
    });

-    await api.downloadBudget(this.#config.budget);    
-    console.log(`Importing ${this.#transactions.length} transactions`);
-    const result = await api.importTransactions(this.#config.account, this.#transactions);
+    await api.downloadBudget(this.#config.budget);
+    
+    const accounts: ActualAccount[] = await api.getAccounts();
+    const payees: ActualPayee[] = await api.getPayees();
+
+    const toImport = transactions.map(t => this.#map(t, accounts, payees))
+
+    const output: Awaited<ReturnType<typeof api.importTransactions>> = { added: [], updated: [], errors: [] };
+    
+
+    if (this.#dryRun) for (const transaction of toImport ){
+      console.log(`${transaction.imported_payee} ::: ${transaction.notes} ::: ${transaction.amount}`);
+    }
+
+    else for (const transaction of toImport) {      
+      const result = await api.importTransactions(transaction.account, [transaction]);
+
+      output.added.push(...result.added);
+      output.updated.push(...result.updated);
+      output.errors?.push(...(result.errors ?? []));     
+      
+      await sleep(100);
+    }
+
    await api.shutdown();    

-    return result;
+    return output;
  }
+}
+
+async function sleep(ms: number): Promise<void> {
+  return new Promise(resolve => setTimeout(resolve, ms));
 }
--- a/src/types/config.ts
+++ b/src/types/config.ts
@@ -18,8 +18,8 @@ export type ParserConfig = {

 export type ServerConfig = {
  budget: string;
-  account: string;
  url: string;
  password: string;
  data: string;
+  accountAliases: Record<string, string>;  
 };
--- a/src/types/transaction.ts
+++ b/src/types/transaction.ts
@@ -1,14 +1,9 @@
 export type Transaction = {
-  id?: string;  
-  date: string;
-  amount?: number;
-  payee?: string;
-  payee_name?: string;
-  imported_payee?: string;
-  category_id?: string;
-  notes?: string;
-  imported_id?: string;
-  transfer_id?: string;
-  cleared?: boolean;
-  subtransactions?: Transaction[];
+  kind: "regular" | "transfer";
+  date: string;  
+  from: string;
+  to: string;  
+  amount: number;
+  id?: string;
+  title?: string;  
 };
--- a/src/util/code.ts
+++ b/src/util/code.ts
@@ -2,8 +2,8 @@ const standardContext = {
  
 };

-export function jsMapper<I, O>(code: string, context: Record<string, unknown>): (task: I) => O {
+export function js2<I1, I2, O>(code: string, i1: string, i2: string, context: Record<string, unknown> = {}): (i1: I1, i2: I2) => O {
  const ctx = { ...standardContext, ...context };
-  const filter = new Function('$', ...Object.keys(ctx), code);
-  return (task: I) => filter(task, ...Object.values(ctx));
+  const filter = new Function(i1, i2, ...Object.keys(ctx), code);
+  return (i1: I1, i2: I2) => filter(i1, i2, ...Object.values(ctx));
 }
--- a/src/util/parser.ts
+++ b/src/util/parser.ts
@@ -0,0 +1,66 @@
+export function mapCombine<T extends object, E>(
+  items: T[],
+  property: keyof T,
+  map: (item: T) => E,
+  combine: (a: T, b: T) => E | undefined,  
+): E[] {
+  // Helper function to check if a value is nullish
+  const isNullish = (value: any): boolean => 
+    value === undefined || 
+    value === null || 
+    (typeof value === 'string' && value.trim() === '');
+
+  const result: Array<{
+    value: T | E;
+    combined: boolean;
+    skip: boolean;
+  }> = items.map(item => ({
+    value: item,
+    combined: false,
+    skip: false
+  }));
+  
+  for (let i = 0; i < result.length; i++) {
+    if (result[i].combined || result[i].skip) continue;
+    
+    const propValueI = (result[i].value as T)[property];
+    if (isNullish(propValueI)) continue; // Skip items with nullish property values
+    
+    for (let j = i + 1; j < result.length; j++) {
+      if (result[j].skip) continue;
+      
+      const propValueJ = (result[j].value as T)[property];
+      if (isNullish(propValueJ)) continue; // Skip if second item has nullish property
+      
+      if (propValueI === propValueJ) {
+        const combinedValue = combine(result[i].value as T, result[j].value as T);
+        
+        if (combinedValue !== undefined) {
+          result[i].value = combinedValue;
+          result[i].combined = true;
+          result[j].skip = true;
+          break;
+        }
+      }
+    }
+  }
+  
+  return result
+    .filter(item => !item.skip)
+    .map(item => item.combined ? item.value as E : map(item.value as T));
+}
+
+
+export function parseAmount(input?: string): number|undefined {    
+  if (input === undefined) {
+    return undefined;
+  }  
+
+  const v = Number.parseFloat(input.replaceAll(",", "."));  
+  
+  if (isNaN(v)) {
+    return undefined;
+  }
+
+  return v;
+}