From 08dba9c51f8a9d295408e68c34fbd6fd2e1cef68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bart=C5=82omiej=20Pluta?= Date: Tue, 6 May 2025 16:10:13 +0200 Subject: [PATCH] Add support for 2-stage import with web server --- package-lock.json | 389 ++++++++++++++++++++++++++++++++++++++++++++ package.json | 5 + package.nix | 7 +- src/cli/index.ts | 4 +- src/runner/index.ts | 54 ++++-- src/server/index.ts | 128 +++------------ views/index.pug | 222 +++++++++++++++++++++++++ 7 files changed, 688 insertions(+), 121 deletions(-) create mode 100644 views/index.pug diff --git a/package-lock.json b/package-lock.json index 8e20582..487a37c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -18,6 +18,7 @@ "iconv-lite": "^0.6.3", "multer": "^1.4.5-lts.2", "papaparse": "^5.5.2", + "pug": "^3.0.3", "yaml": "^2.7.1" }, "bin": { @@ -57,6 +58,52 @@ "uuid": "^9.0.0" } }, + "node_modules/@babel/helper-string-parser": { + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.27.1.tgz", + "integrity": "sha512-qMlSxKbpRlAridDExk92nSobyDdpPijUq2DW6oDnUqd0iOGxmQjyqhMIihI9+zv4LPyZdRje2cavWPbCbWm3eA==", + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-identifier": { + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.27.1.tgz", + "integrity": "sha512-D2hP9eA+Sqx1kBZgzxZh0y1trbuU+JoDkiEwqhQ36nodYqJwyEIhPSdMNd7lOm/4io72luTPWH20Yda0xOuUow==", + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/parser": { + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.27.1.tgz", + "integrity": "sha512-I0dZ3ZpCrJ1c04OqlNsQcKiZlsrXf/kkE4FXzID9rIOYICsAbA8mMDzhW/luRNAHdCNt7os/u8wenklZDlUVUQ==", + "license": "MIT", + "dependencies": { + "@babel/types": "^7.27.1" + }, + "bin": { + "parser": "bin/babel-parser.js" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@babel/types": { + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.27.1.tgz", + "integrity": "sha512-+EzkxvLNfiUeKMgy/3luqfsCWFRXLb7U6wNQTk60tovuckwB15B191tJWvpp4HjiQWdJkCxO3Wbvc6jlk3Xb2Q==", + "license": "MIT", + "dependencies": { + "@babel/helper-string-parser": "^7.27.1", + "@babel/helper-validator-identifier": "^7.27.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, "node_modules/@esbuild/aix-ppc64": { "version": "0.25.3", "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.25.3.tgz", @@ -647,6 +694,18 @@ "node": ">= 0.6" } }, + "node_modules/acorn": { + "version": "7.4.1", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-7.4.1.tgz", + "integrity": "sha512-nQyp0o1/mNdbTO1PO6kHkwSrmgZ0MT/jCCpNiwbUjGoRN4dlBhqJtoQuCnEOKzgTVwg0ZWiCoQy6SxMebQVh8A==", + "license": "MIT", + "bin": { + "acorn": "bin/acorn" + }, + "engines": { + "node": ">=0.4.0" + } + }, "node_modules/anymatch": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.3.tgz", @@ -677,6 +736,30 @@ "node": ">=8" } }, + "node_modules/asap": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/asap/-/asap-2.0.6.tgz", + "integrity": "sha512-BSHWgDSAiKs50o2Re8ppvp3seVHXSRM44cdSsT9FfNEUUZLOGWVCsiWaRPWM1Znn+mqZ1OfVZ3z3DWEzSp7hRA==", + "license": "MIT" + }, + "node_modules/assert-never": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/assert-never/-/assert-never-1.4.0.tgz", + "integrity": "sha512-5oJg84os6NMQNl27T9LnZkvvqzvAnHu03ShCnoj6bsJwS7L8AO4lf+C/XjK/nvzEqQB744moC6V128RucQd1jA==", + "license": "MIT" + }, + "node_modules/babel-walk": { + "version": "3.0.0-canary-5", + "resolved": "https://registry.npmjs.org/babel-walk/-/babel-walk-3.0.0-canary-5.tgz", + "integrity": "sha512-GAwkz0AihzY5bkwIY5QDR+LvsRQgB/B+1foMPvi0FZPMl5fjD7ICiznUiBdLYMH1QYe6vqu4gWYytZOccLouFw==", + "license": "MIT", + "dependencies": { + "@babel/types": "^7.9.6" + }, + "engines": { + "node": ">= 10.0.0" + } + }, "node_modules/base64-js": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", @@ -867,6 +950,15 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/character-parser": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/character-parser/-/character-parser-2.2.0.tgz", + "integrity": "sha512-+UqJQjFEFaTAs3bNsF2j2kEN1baG/zghZbdqoYEDxGZtJo9LBzl1A+m0D4n3qKx8N2FNv8/Xp6yV9mQmBuptaw==", + "license": "MIT", + "dependencies": { + "is-regex": "^1.0.3" + } + }, "node_modules/chokidar": { "version": "3.6.0", "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.6.0.tgz", @@ -928,6 +1020,16 @@ "typedarray": "^0.0.6" } }, + "node_modules/constantinople": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/constantinople/-/constantinople-4.0.1.tgz", + "integrity": "sha512-vCrqcSIq4//Gx74TXXCGnHpulY1dskqLTFGDmhrGxzeXL8lF8kvXv6mpNWlJj1uD4DW23D4ljAqbY4RRaaUZIw==", + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.6.0", + "@babel/types": "^7.6.1" + } + }, "node_modules/content-disposition": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-1.0.0.tgz", @@ -1054,6 +1156,12 @@ "node": ">=8" } }, + "node_modules/doctypes": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/doctypes/-/doctypes-1.1.0.tgz", + "integrity": "sha512-LLBi6pEqS6Do3EKQ3J0NqHWV5hhb78Pi8vvESYwyOy2c31ZEZVdtitdzsQsKb7878PEERhzUk0ftqGhG6Mz+pQ==", + "license": "MIT" + }, "node_modules/dunder-proto": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", @@ -1495,6 +1603,21 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/has-tostringtag": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz", + "integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==", + "license": "MIT", + "dependencies": { + "has-symbols": "^1.0.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/hasown": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", @@ -1599,6 +1722,31 @@ "node": ">=8" } }, + "node_modules/is-core-module": { + "version": "2.16.1", + "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.16.1.tgz", + "integrity": "sha512-UfoeMA6fIJ8wTYFEUjelnaGI67v6+N7qXJEvQuIGa99l4xsCruSYOVSQ0uPANn4dAzm8lkYPaKLrrijLq7x23w==", + "license": "MIT", + "dependencies": { + "hasown": "^2.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-expression": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/is-expression/-/is-expression-4.0.0.tgz", + "integrity": "sha512-zMIXX63sxzG3XrkHkrAPvm/OVZVSCPNkwMHU8oTX7/U3AL78I0QXCEICXUM13BIa8TYGZ68PiTKfQz3yaTNr4A==", + "license": "MIT", + "dependencies": { + "acorn": "^7.1.1", + "object-assign": "^4.1.1" + } + }, "node_modules/is-extglob": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", @@ -1638,12 +1786,52 @@ "integrity": "sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ==", "license": "MIT" }, + "node_modules/is-regex": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.2.1.tgz", + "integrity": "sha512-MjYsKHO5O7mCsmRGxWcLWheFqN9DJ/2TmngvjKXihe6efViPqc274+Fx/4fYj/r03+ESvBdTXK0V6tA3rgez1g==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "gopd": "^1.2.0", + "has-tostringtag": "^1.0.2", + "hasown": "^2.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/isarray": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==", "license": "MIT" }, + "node_modules/js-stringify": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/js-stringify/-/js-stringify-1.0.2.tgz", + "integrity": "sha512-rtS5ATOo2Q5k1G+DADISilDA6lv79zIiwFd6CcjuIxGKLFm5C+RLImRscVap9k55i+MOZwgliw+NejvkLuGD5g==", + "license": "MIT" + }, + "node_modules/jstransformer": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/jstransformer/-/jstransformer-1.0.0.tgz", + "integrity": "sha512-C9YK3Rf8q6VAPDCCU9fnqo3mAfOH6vUGnMcP4AQAYIEpWtfGLpwOTmZ+igtdK5y+VvI2n3CyYSzy4Qh34eq24A==", + "license": "MIT", + "dependencies": { + "is-promise": "^2.0.0", + "promise": "^7.0.1" + } + }, + "node_modules/jstransformer/node_modules/is-promise": { + "version": "2.2.2", + "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-2.2.2.tgz", + "integrity": "sha512-+lP4/6lKUBfQjZ2pdxThZvLUAafmZb8OAxFb8XXtiQmS35INgr85hdOGoEs124ez1FCnZJt6jau/T+alh58QFQ==", + "license": "MIT" + }, "node_modules/math-intrinsics": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", @@ -1977,6 +2165,12 @@ "node": ">= 0.8" } }, + "node_modules/path-parse": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz", + "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==", + "license": "MIT" + }, "node_modules/path-to-regexp": { "version": "8.2.0", "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-8.2.0.tgz", @@ -2054,6 +2248,15 @@ "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==", "license": "MIT" }, + "node_modules/promise": { + "version": "7.3.1", + "resolved": "https://registry.npmjs.org/promise/-/promise-7.3.1.tgz", + "integrity": "sha512-nolQXZ/4L+bP/UGlkfaIujX9BKxGwmQ9OT4mOt5yvy8iK1h3wqTEJCijzGANTCCl9nWjY41juyAn2K3Q1hLLTg==", + "license": "MIT", + "dependencies": { + "asap": "~2.0.3" + } + }, "node_modules/proxy-addr": { "version": "2.0.7", "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", @@ -2067,6 +2270,130 @@ "node": ">= 0.10" } }, + "node_modules/pug": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/pug/-/pug-3.0.3.tgz", + "integrity": "sha512-uBi6kmc9f3SZ3PXxqcHiUZLmIXgfgWooKWXcwSGwQd2Zi5Rb0bT14+8CJjJgI8AB+nndLaNgHGrcc6bPIB665g==", + "license": "MIT", + "dependencies": { + "pug-code-gen": "^3.0.3", + "pug-filters": "^4.0.0", + "pug-lexer": "^5.0.1", + "pug-linker": "^4.0.0", + "pug-load": "^3.0.0", + "pug-parser": "^6.0.0", + "pug-runtime": "^3.0.1", + "pug-strip-comments": "^2.0.0" + } + }, + "node_modules/pug-attrs": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pug-attrs/-/pug-attrs-3.0.0.tgz", + "integrity": "sha512-azINV9dUtzPMFQktvTXciNAfAuVh/L/JCl0vtPCwvOA21uZrC08K/UnmrL+SXGEVc1FwzjW62+xw5S/uaLj6cA==", + "license": "MIT", + "dependencies": { + "constantinople": "^4.0.1", + "js-stringify": "^1.0.2", + "pug-runtime": "^3.0.0" + } + }, + "node_modules/pug-code-gen": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/pug-code-gen/-/pug-code-gen-3.0.3.tgz", + "integrity": "sha512-cYQg0JW0w32Ux+XTeZnBEeuWrAY7/HNE6TWnhiHGnnRYlCgyAUPoyh9KzCMa9WhcJlJ1AtQqpEYHc+vbCzA+Aw==", + "license": "MIT", + "dependencies": { + "constantinople": "^4.0.1", + "doctypes": "^1.1.0", + "js-stringify": "^1.0.2", + "pug-attrs": "^3.0.0", + "pug-error": "^2.1.0", + "pug-runtime": "^3.0.1", + "void-elements": "^3.1.0", + "with": "^7.0.0" + } + }, + "node_modules/pug-error": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/pug-error/-/pug-error-2.1.0.tgz", + "integrity": "sha512-lv7sU9e5Jk8IeUheHata6/UThZ7RK2jnaaNztxfPYUY+VxZyk/ePVaNZ/vwmH8WqGvDz3LrNYt/+gA55NDg6Pg==", + "license": "MIT" + }, + "node_modules/pug-filters": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/pug-filters/-/pug-filters-4.0.0.tgz", + "integrity": "sha512-yeNFtq5Yxmfz0f9z2rMXGw/8/4i1cCFecw/Q7+D0V2DdtII5UvqE12VaZ2AY7ri6o5RNXiweGH79OCq+2RQU4A==", + "license": "MIT", + "dependencies": { + "constantinople": "^4.0.1", + "jstransformer": "1.0.0", + "pug-error": "^2.0.0", + "pug-walk": "^2.0.0", + "resolve": "^1.15.1" + } + }, + "node_modules/pug-lexer": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/pug-lexer/-/pug-lexer-5.0.1.tgz", + "integrity": "sha512-0I6C62+keXlZPZkOJeVam9aBLVP2EnbeDw3An+k0/QlqdwH6rv8284nko14Na7c0TtqtogfWXcRoFE4O4Ff20w==", + "license": "MIT", + "dependencies": { + "character-parser": "^2.2.0", + "is-expression": "^4.0.0", + "pug-error": "^2.0.0" + } + }, + "node_modules/pug-linker": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/pug-linker/-/pug-linker-4.0.0.tgz", + "integrity": "sha512-gjD1yzp0yxbQqnzBAdlhbgoJL5qIFJw78juN1NpTLt/mfPJ5VgC4BvkoD3G23qKzJtIIXBbcCt6FioLSFLOHdw==", + "license": "MIT", + "dependencies": { + "pug-error": "^2.0.0", + "pug-walk": "^2.0.0" + } + }, + "node_modules/pug-load": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pug-load/-/pug-load-3.0.0.tgz", + "integrity": "sha512-OCjTEnhLWZBvS4zni/WUMjH2YSUosnsmjGBB1An7CsKQarYSWQ0GCVyd4eQPMFJqZ8w9xgs01QdiZXKVjk92EQ==", + "license": "MIT", + "dependencies": { + "object-assign": "^4.1.1", + "pug-walk": "^2.0.0" + } + }, + "node_modules/pug-parser": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/pug-parser/-/pug-parser-6.0.0.tgz", + "integrity": "sha512-ukiYM/9cH6Cml+AOl5kETtM9NR3WulyVP2y4HOU45DyMim1IeP/OOiyEWRr6qk5I5klpsBnbuHpwKmTx6WURnw==", + "license": "MIT", + "dependencies": { + "pug-error": "^2.0.0", + "token-stream": "1.0.0" + } + }, + "node_modules/pug-runtime": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/pug-runtime/-/pug-runtime-3.0.1.tgz", + "integrity": "sha512-L50zbvrQ35TkpHwv0G6aLSuueDRwc/97XdY8kL3tOT0FmhgG7UypU3VztfV/LATAvmUfYi4wNxSajhSAeNN+Kg==", + "license": "MIT" + }, + "node_modules/pug-strip-comments": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/pug-strip-comments/-/pug-strip-comments-2.0.0.tgz", + "integrity": "sha512-zo8DsDpH7eTkPHCXFeAk1xZXJbyoTfdPlNR0bK7rpOMuhBYb0f5qUVCO1xlsitYd3w5FQTK7zpNVKb3rZoUrrQ==", + "license": "MIT", + "dependencies": { + "pug-error": "^2.0.0" + } + }, + "node_modules/pug-walk": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/pug-walk/-/pug-walk-2.0.0.tgz", + "integrity": "sha512-yYELe9Q5q9IQhuvqsZNwA5hfPkMJ8u92bQLIMcsMxf/VADjNtEYptU+inlufAFYcWdHlwNfZOEnOOQrZrcyJCQ==", + "license": "MIT" + }, "node_modules/pump": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.2.tgz", @@ -2196,6 +2523,26 @@ "node": ">=8.10.0" } }, + "node_modules/resolve": { + "version": "1.22.10", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.10.tgz", + "integrity": "sha512-NPRy+/ncIMeDlTAsuqwKIiferiawhefFJtkNSW0qZJEqMEb+qBt/77B/jGeeek+F0uOeN05CDa6HXbbIgtVX4w==", + "license": "MIT", + "dependencies": { + "is-core-module": "^2.16.0", + "path-parse": "^1.0.7", + "supports-preserve-symlinks-flag": "^1.0.0" + }, + "bin": { + "resolve": "bin/resolve" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/resolve-pkg-maps": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/resolve-pkg-maps/-/resolve-pkg-maps-1.0.0.tgz", @@ -2506,6 +2853,18 @@ "node": ">=0.10.0" } }, + "node_modules/supports-preserve-symlinks-flag": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz", + "integrity": "sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/tar-fs": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.2.tgz", @@ -2570,6 +2929,12 @@ "node": ">=0.6" } }, + "node_modules/token-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/token-stream/-/token-stream-1.0.0.tgz", + "integrity": "sha512-VSsyNPPW74RpHwR8Fc21uubwHY7wMDeJLys2IX5zJNih+OnAnaifKHo+1LHT7DAdloQ7apeaaWg8l7qnf/TnEg==", + "license": "MIT" + }, "node_modules/tsc-alias": { "version": "1.8.15", "resolved": "https://registry.npmjs.org/tsc-alias/-/tsc-alias-1.8.15.tgz", @@ -2711,6 +3076,15 @@ "node": ">= 0.8" } }, + "node_modules/void-elements": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/void-elements/-/void-elements-3.1.0.tgz", + "integrity": "sha512-Dhxzh5HZuiHQhbvTW9AMetFfBHDMYpo23Uo9btPXgdYP+3T5S+p+jgNy7spra+veYhBP2dCSgxR/i2Y02h5/6w==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/web-streams-polyfill": { "version": "3.3.3", "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz", @@ -2720,6 +3094,21 @@ "node": ">= 8" } }, + "node_modules/with": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/with/-/with-7.0.2.tgz", + "integrity": "sha512-RNGKj82nUPg3g5ygxkQl0R937xLyho1J24ItRCBTr/m1YnZkzJy1hUiHUJrc/VlsDQzsCnInEGSg3bci0Lmd4w==", + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.9.6", + "@babel/types": "^7.9.6", + "assert-never": "^1.2.1", + "babel-walk": "3.0.0-canary-5" + }, + "engines": { + "node": ">= 10.0.0" + } + }, "node_modules/wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", diff --git a/package.json b/package.json index 7faabab..eb3b55f 100644 --- a/package.json +++ b/package.json @@ -7,6 +7,10 @@ "start": "tsx src/index.ts", "build": "tsc && tsc-alias" }, + "files": [ + "dist", + "views" + ], "author": "Bartłomiej Pluta ", "license": "ISC", "bin": { @@ -28,6 +32,7 @@ "iconv-lite": "^0.6.3", "multer": "^1.4.5-lts.2", "papaparse": "^5.5.2", + "pug": "^3.0.3", "yaml": "^2.7.1" } } diff --git a/package.nix b/package.nix index 977f002..97b4e0f 100644 --- a/package.nix +++ b/package.nix @@ -7,5 +7,10 @@ buildNpmPackage { pname = "actual-importer"; version = "0.0.1"; src = ./.; - npmDepsHash = "sha256-ayfvTn8FVo7/K/Gy7oTDQiXDewvrU7B5JsNLHfrxibQ="; + npmDepsHash = "sha256-QqSZJuQLPll3qFi5Lv4kbQo+548l3VKgx073WLNXMsw="; + + postInstall = '' + mkdir -p $out/views + cp -r ${./views}/* $out/views/ + ''; } diff --git a/src/cli/index.ts b/src/cli/index.ts index f1f0f2b..72ad279 100644 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -2,7 +2,7 @@ import fs from "fs"; import { program } from "commander"; import { AddOptions, ImportOptions } from "@/types/cli"; import { loadConfig } from "./config"; -import { submitTransactions } from "@/runner"; +import { loadTransactions } from "@/runner"; import { serve } from "@/server"; import { SubmitOptions } from "@/backend"; @@ -23,7 +23,7 @@ function doSubmit(parseOpts: (o: O) => Sub const opts: SubmitOptions = options.dryRun ? { mode: 'dry-run' } : parseOpts(options); - await submitTransactions(fs.createReadStream(file), profile, server, config, opts); + await loadTransactions(fs.createReadStream(file), profile, server, config, opts); } } diff --git a/src/runner/index.ts b/src/runner/index.ts index e5afb45..845d2eb 100644 --- a/src/runner/index.ts +++ b/src/runner/index.ts @@ -6,13 +6,17 @@ import { Config } from "@/types/config"; import { Readable } from "stream"; import { Transaction } from "@/types/transaction"; -export type ImportResult = { - transactions: Transaction[]; - result: ActualImportResult; +export type PrepareResult = { + transactions: Transaction[]; skipped: string[][]; }; -export const submitTransactions = async (stream: Readable, profile: string, server: string, config: Config, opts: SubmitOptions): Promise => new Promise((resolve, reject) => { +export type ImportResult = PrepareResult & { + result: ActualImportResult; +}; + + +export const prepareTransactions = async (stream: Readable, profile: string, server: string, config: Config): Promise => new Promise((resolve, reject) => { const profileConfig = config.profiles[profile]; if (!profileConfig) { @@ -25,8 +29,7 @@ export const submitTransactions = async (stream: Readable, profile: string, serv } const parser = createParser(profileConfig, serverConfig); - - const actualServer = new Actual(serverConfig); + const skipped: string[][] = []; const handleRow = async (data: string[]) => { @@ -40,18 +43,15 @@ export const submitTransactions = async (stream: Readable, profile: string, serv const handleClose = async () => { try { const transactions = await parser.reconcile(); - const result = await actualServer.load(transactions, opts); - + resolve({ - transactions, - result, + transactions, skipped - }); - - } catch (e) { + }); + } catch (e: unknown) { console.error(e); - reject(e); - } + reject(e); + } }; stream @@ -59,4 +59,26 @@ export const submitTransactions = async (stream: Readable, profile: string, serv .pipe(Papa.parse(Papa.NODE_STREAM_INPUT, profileConfig.csv ?? parser.csvConfig)) .on('data', handleRow) .on('close', handleClose); -}); \ No newline at end of file +}); + +export const submitTransactions = async (transactions: Transaction[], server: string, config: Config, opts: SubmitOptions): Promise => { + const serverConfig = config.servers[server]; + + if(!serverConfig) { + throw new Error(`Unknown server: ${server}`); + } + + const actualServer = new Actual(serverConfig); + return await actualServer.load(transactions, opts); +}; + +export const loadTransactions = async (stream: Readable, profile: string, server: string, config: Config, opts: SubmitOptions): Promise => { + const prepared = await prepareTransactions(stream, profile, server, config) + + const result = await submitTransactions(prepared.transactions, server, config, opts); + + return { + ...prepared, + result + } +}; diff --git a/src/server/index.ts b/src/server/index.ts index 6c7ba2b..0f0c6f1 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -1,130 +1,54 @@ -import { SubmitOptions } from "@/backend"; -import { ImportResult, submitTransactions } from "@/runner"; +import { prepareTransactions, submitTransactions } from "@/runner"; import { Config } from "@/types/config"; import express from "express"; import multer from "multer"; import { Readable } from "stream"; +import path from 'path'; + export function serve(config: Config, port: number) { const app = express(); + + app.set('view engine', 'pug'); + app.set('views', path.join(__dirname, '../../views')); + + app.use(express.static('public')); + const upload = multer({ storage: multer.memoryStorage() }); app.get("/", (req, res) => { - res.send(` - - -

Import transactions

-
-

- - -

-

- - -

-

- - -

-

- - -

-

- - -

-

- - -

-

- -

-
- - - `); + res.render('index', { + profiles: Object.keys(config.profiles), + servers: Object.keys(config.servers), + defaultProfile: config.defaultProfile, + defaultServer: config.defaultServer + }); }); - app.post("/import", upload.single('file'), async (req, res) => { + app.post("/prepare", upload.single("file"), async (req, res) => { if (!req.file) { throw new Error("No file to upload"); } - const { profile, server, learn, transfers, mode } = req.body; + const { profile, server } = req.body; const stream = new Readable(); stream.push(req.file.buffer); stream.push(null); - const opts: SubmitOptions = { - mode: mode as 'import'|'add', - learnCategories: readCheckbox(learn), - runTransfers: readCheckbox(transfers) - }; + const response = await prepareTransactions(stream, profile, server, config); - const result = await submitTransactions(stream, profile, server, config, opts); - - res.send(formatResult(result)); + res.send(response); }); + app.post("/submit", express.json(), async (req, res) => { + const { transactions, opts, server } = req.body; + const response = await submitTransactions(transactions, server, config, opts); + res.send(response); + }); + + app.listen(port, () => { console.log(`Server running on ${port} port`); }); } - -function readCheckbox(value: string|undefined): boolean { - return ["on", "true"].includes(value?.toLowerCase() ?? ""); -} - -function formatResult(result: ImportResult): string { - return ` - - -

Import Result

-

Considered transactions (${result.transactions.length})

- - - - - - - - - - - - - - ${result.transactions.map(t => ``)} - -
IDKindDateFromToAmountTitle
${t.id}${t.kind}${t.date}${t.from}${t.to}${t.amount}${t.title}
- -

Import status

-

-

    -
  • Added: ${result.result.added.length}
    • -
  • Updated: ${result.result.updated.length}
    • -
  • Errors: ${result.result?.errors?.length ?? 0}
    • -
-

- ${(result.result.errors?.length ?? 0) > 0 - ? `

Errors:

    ${result.result.errors?.map(e => `
  • ${e.message}
    • `)}

` - : ""} - -

Skipped CSV rows

-

The ::: is CSV field delimiter

- 
${result.skipped.map(d => d.join(" ::: ")).join("\n")}
- - - `; -} \ No newline at end of file diff --git a/views/index.pug b/views/index.pug new file mode 100644 index 0000000..9f81e0b --- /dev/null +++ b/views/index.pug @@ -0,0 +1,222 @@ +html + head + title Actual Importer + + body + h1 Import transactions + + form#prepareForm + p + label(for="file") CSV File: + input(type="file" name="file" required) + + p + label(for="profile") Profile: + select(name="profile" required) + each profile in profiles + option(value=profile selected=(defaultProfile === profile))= profile + + p + label(for="server") Server: + select(name="server" required) + each server in servers + option(value=server selected=(defaultServer === server))= server + + button(type="submit") Load + + div#prepare + + div#result + + script. + const form = document.querySelector("#prepareForm"); + + function renderTable(transactions) { + const div = document.createElement('div'); + div.innerHTML = ` +

Pending transactions (${transactions.length})

+ `; + const table = document.createElement('table'); + const thead = document.createElement('thead'); + const tr = document.createElement('tr'); + tr.innerHTML = ` + # + Import + ID + Kind + Date + From + To + Amount + Title + `; + + thead.appendChild(tr); + table.appendChild(thead); + + const tbody = document.createElement('tbody'); + transactions.forEach((transaction, index) => { + const row = document.createElement('tr'); + + row.innerHTML = ` + ${index+1} + + ${transaction.id} + ${transaction.kind} + ${transaction.date} + ${transaction.from} + ${transaction.to} + ${transaction.amount} + ${transaction.title} + `; + + tbody.appendChild(row); + }); + + table.appendChild(tbody); + div.appendChild(table); + return div; + } + + function renderSkipped(skipped) { + const div = document.createElement('div'); + div.innerHTML = ` +

Skipped CSV rows

+

The ::: is CSV field delimiter

+ 
${skipped.map(d => d.join(" ::: ")).join("\n")}
+ ` + + return div; + } + + function renderResult(result) { + const resultWrapper = document.querySelector('#result'); + resultWrapper.innerHTML = ` +

Import status

+

+

    +
  • Added: ${result.added.length}
    • +
  • Updated: ${result.updated.length}
    • +
  • Errors: ${result?.errors?.length ?? 0}
    • +
+ ${(result.errors?.length ?? 0) > 0 + ? `

Errors:

    ${result.errors?.map(e => `
  • ${e.message}
    • `)}

` + : ""} +

+ `; + } + + async function submitTransactions(config, transactions, opts) { + const filtered = transactions.filter((transaction, index) => !!document.querySelector(`#transaction${index}`).checked); + + if (filtered.length === 0) { + return; + } + + try { + const response = await fetch("/submit", { + method: "POST", + headers: { + 'Content-Type': 'application/json' + }, + body: JSON.stringify({ + server: config.get('server'), + transactions: filtered, + opts + }), + }); + + const result = await response.json(); + renderResult(result); + } catch (e) { + console.error(e); + } + } + + function renderPrepare(config, data) { + document.querySelector('#result').innerHTML = ''; + const prepare = document.querySelector("#prepare"); + prepare.innerHTML = ''; + + const title = document.createElement("h2"); + title.innerHTML = "Prepare transactions to submit"; + + prepare.appendChild(title); + prepare.appendChild(renderTable(data.transactions)); + prepare.appendChild(renderSkipped(data.skipped)); + + const addForm = document.createElement("form"); + addForm.innerHTML = ` +
+ Add +

+ + +

+ +

+ + +

+ + +
+ `; + + addForm.addEventListener("submit", async (e) => { + e.preventDefault(); + const options = new FormData(addForm); + const opts = { + mode: 'add', + learnCategories: !!options.get('learn'), + runTransfers: !!options.get('transfers') + }; + + await submitTransactions(config, data.transactions, opts); + prepare.innerHTML = ''; + }); + + const importForm = document.createElement("form"); + importForm.innerHTML = ` +
+ Import + +
+ `; + + importForm.addEventListener("submit", async (e) => { + e.preventDefault(); + await submitTransactions(config, data.transactions, { mode: 'import' }); + prepare.innerHTML = ''; + }); + + if (data.transactions.length > 0) { + prepare.appendChild(addForm); + prepare.appendChild(importForm); + } else { + const message = document.createElement('p'); + message.innerHTML = ` + No transactions to import + `; + prepare.appendChild(message); + } + } + + async function handlePrepare(e) { + e.preventDefault(); + + try { + const config = new FormData(form); + const response = await fetch("/prepare", { + method: "POST", + body: config, + }); + + const data = await response.json(); + renderPrepare(config, data); + } catch (e) { + console.error(e); + } + } + + form.addEventListener("submit", handlePrepare); \ No newline at end of file