From 58b2d2c629a24b702cf6805bc0d114cf3cf70e30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bart=C5=82omiej=20Pluta?= Date: Tue, 1 Apr 2025 20:31:16 +0200 Subject: [PATCH] Replace broken windows-1250 package with iconv-lite --- package-lock.json | 26 ++++++++++++++----- package.json | 2 +- package.nix | 2 +- src/csv/decoder.ts | 12 --------- src/csv/index.ts | 1 - src/csv/pipeline.ts | 9 ------- src/runner/index.ts | 63 +++++++++++++++++++++++++-------------------- 7 files changed, 56 insertions(+), 59 deletions(-) delete mode 100644 src/csv/decoder.ts delete mode 100644 src/csv/index.ts delete mode 100644 src/csv/pipeline.ts diff --git a/package-lock.json b/package-lock.json index b3831d1..f51f981 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,8 +12,8 @@ "@actual-app/api": "^25.3.1", "@types/papaparse": "^5.3.15", "commander": "^13.1.0", + "iconv-lite": "^0.6.3", "papaparse": "^5.5.2", - "windows-1250": "^3.0.4", "yaml": "^2.7.1" }, "bin": { @@ -557,6 +557,18 @@ "integrity": "sha512-MnG7N936zcKTco4Jd2PX2U96Kf9PxygAPKBug+74LHzmHXmceN16MmRcdgZv+DGef/S9YvQAfRsNCn4cjf9yyQ==", "license": "(BSD-3-Clause AND Apache-2.0)" }, + "node_modules/iconv-lite": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", + "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/ieee754": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", @@ -1006,6 +1018,12 @@ ], "license": "MIT" }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "license": "MIT" + }, "node_modules/semver": { "version": "7.7.1", "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.1.tgz", @@ -1243,12 +1261,6 @@ "node": ">= 8" } }, - "node_modules/windows-1250": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/windows-1250/-/windows-1250-3.0.4.tgz", - "integrity": "sha512-DoHOT9o9N/TwKoLkLWe/pedZvZqhif3GY5YagTK/c8ONbu6K8tzt7lnH6do7oACmwU+q+6f2IwUjGyBzRZJXbA==", - "license": "MIT" - }, "node_modules/wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", diff --git a/package.json b/package.json index 0df55e0..49fad27 100644 --- a/package.json +++ b/package.json @@ -22,8 +22,8 @@ "@actual-app/api": "^25.3.1", "@types/papaparse": "^5.3.15", "commander": "^13.1.0", + "iconv-lite": "^0.6.3", "papaparse": "^5.5.2", - "windows-1250": "^3.0.4", "yaml": "^2.7.1" } } diff --git a/package.nix b/package.nix index 9157f87..47497db 100644 --- a/package.nix +++ b/package.nix @@ -7,5 +7,5 @@ buildNpmPackage { pname = "actual-importer"; version = "0.0.1"; src = ./.; - npmDepsHash = "sha256-ovYlyRG4EllqpDKWRrPzxO/A9rTwKs1WtnlAqOgKeaI="; + npmDepsHash = "sha256-u880X9C5s69nFU0uMt6pRLRGgCui0Pwx1K9lrOroPYw="; } diff --git a/src/csv/decoder.ts b/src/csv/decoder.ts deleted file mode 100644 index 0ed1f74..0000000 --- a/src/csv/decoder.ts +++ /dev/null @@ -1,12 +0,0 @@ -import { decode } from "windows-1250"; -import { Transform } from 'stream'; - -export default new Transform({ - transform(chunk, encoding, callback) { - try { - callback(null, decode(chunk)); - } catch(error: any) { - callback(error); - } - }, -}); \ No newline at end of file diff --git a/src/csv/index.ts b/src/csv/index.ts deleted file mode 100644 index 8c15717..0000000 --- a/src/csv/index.ts +++ /dev/null @@ -1 +0,0 @@ -export { default as openCsv } from "./pipeline"; \ No newline at end of file diff --git a/src/csv/pipeline.ts b/src/csv/pipeline.ts deleted file mode 100644 index 7eec0c6..0000000 --- a/src/csv/pipeline.ts +++ /dev/null @@ -1,9 +0,0 @@ -import Papa from "papaparse"; -import fs from "fs"; -import decoder from "./decoder"; - -export default function(file: string, config?: Papa.ParseConfig) { - return fs.createReadStream(file) - .pipe(decoder) - .pipe(Papa.parse(Papa.NODE_STREAM_INPUT, config)) -} diff --git a/src/runner/index.ts b/src/runner/index.ts index b282dd1..a42aba3 100644 --- a/src/runner/index.ts +++ b/src/runner/index.ts @@ -1,9 +1,11 @@ +import Papa from "papaparse"; import fs from "fs"; -import { openCsv } from "@/csv"; +import iconv from "iconv-lite"; import { createParser } from "@/parser"; import { Actual } from "@/server"; import { Config } from "@/types/config"; + export function loadTransactions(file: string, profile: string, server: string, config: Config) { const profileConfig = config.profiles[profile]; if (!profileConfig) { @@ -19,33 +21,38 @@ export function loadTransactions(file: string, profile: string, server: string, const actualServer = new Actual(serverConfig); const skipped: string[] = []; - - openCsv(file) - .on('data', async data => { - const transaction = await parser.parseTransaction(profileConfig, data); - - if (transaction === undefined) { - skipped.push(`Skipped ==> ${data.join(" ::: ")}`); - return; - } - actualServer.pushTransaction(transaction); + const handleRow = async (data: string[]) => { + const transaction = await parser.parseTransaction(profileConfig, data); + + if (transaction === undefined) { + skipped.push(`Skipped ==> ${data.join(" ::: ")}`); + return; + } + + actualServer.pushTransaction(transaction); + }; + + const handleClose = () => actualServer.submit() + .then(x => { + console.log(`Inserted: ${x.added.length}`); + console.log(`Updated: ${x.updated.length}`); + console.log(`Errors: ${x.errors?.length}`); + console.log(`Skipped: ${skipped.length}`); + const now = new Date(); + const date = `${now.getFullYear()}-${(now.getMonth()+1).toString().padStart(2, '0')}-${now.getDate().toString().padStart(2, '0')}`; + const time = `${now.getHours().toString().padStart(2, '0')}-${now.getMinutes().toString().padStart(2, '0')}-${now.getSeconds().toString().padStart(2, '0')}` + const filename = `${serverConfig.data}/import.${profile}.${server}.${date}T${time}.json`.replaceAll(/\/+/g, "/"); + + const logContent = `${JSON.stringify(x, undefined, 2)}\n\n\n\n${skipped.join("\n")}` + fs.writeFileSync(filename, logContent); + console.log(`Detailed output written to ${filename} file.`); }) - .on('close', () => actualServer.submit() - .then(x => { - console.log(`Inserted: ${x.added.length}`); - console.log(`Updated: ${x.updated.length}`); - console.log(`Errors: ${x.errors?.length}`); - console.log(`Skipped: ${skipped.length}`); - const now = new Date(); - const date = `${now.getFullYear()}-${(now.getMonth()+1).toString().padStart(2, '0')}-${now.getDate().toString().padStart(2, '0')}`; - const time = `${now.getHours().toString().padStart(2, '0')}-${now.getMinutes().toString().padStart(2, '0')}-${now.getSeconds().toString().padStart(2, '0')}` - const filename = `${serverConfig.data}/import.${profile}.${server}.${date}T${time}.json`.replaceAll(/\/+/g, "/"); - - const logContent = `${JSON.stringify(x, undefined, 2)}\n\n\n\n${skipped.join("\n")}` - fs.writeFileSync(filename, logContent); - console.log(`Detailed output written to ${filename} file.`); - }) - .catch(x => console.error(x)) - ); + .catch(x => console.error(x)) + + fs.createReadStream(file) + .pipe(iconv.decodeStream("win1250")) + .pipe(Papa.parse(Papa.NODE_STREAM_INPUT)) + .on('data', handleRow) + .on('close', handleClose); } \ No newline at end of file