Compare commits

...

2 Commits

Author SHA1 Message Date
95362cc9bc Fix module 2025-01-21 11:31:35 +01:00
acb1bf3065 Add support for time ranged requests 2025-01-21 11:27:12 +01:00
5 changed files with 51 additions and 14 deletions

View File

@@ -13,13 +13,17 @@
flake-utils,
}:
flake-utils.lib.eachDefaultSystem (system: let
pkgs = nixpkgs.legacyPackages.${system};
pkgs = import nixpkgs {inherit system;};
in {
packages = rec {
tauron-scrapper = pkgs.callPackage ./package.nix {};
default = tauron-scrapper;
};
nixosModules.tauron-scrapper = pkgs.callPackage ./module.nix {inherit self;};
});
})
// {
nixosModules = rec {
tauron-scrapper = import ./module.nix self;
default = tauron-scrapper;
};
};
}

View File

@@ -1,14 +1,12 @@
{
self: {
config,
coreutils-full,
formats,
pkgs,
system,
utils,
lib,
self,
...
}:
with lib; let
inherit (pkgs) coreutils-full formats utils;
inherit (utils.systemdUtils.unitOptions) unitOption;
cfg = config.services.tauron-scrapper;
yamlConfig = (formats.yaml {}).generate "tauron-scrapper-config.yaml" cfg.config;
@@ -16,6 +14,21 @@ in {
options.services.tauron-scrapper = {
enable = mkEnableOption "tauron-scrapper";
cliArgs = mkOption {
type = types.attrs;
description = "List of CLI arguments. Do not put '-c' argument here as it will be automatically appended basing on the 'config' option.";
example = {
date = "$(date -d '5 days ago' '+%Y-%m-%d')";
to = "$(date -d 'yesterday' '+%Y-%m-%d')";
};
default = {
date = "$(date -d '5 days ago' '+%Y-%m-%d')";
to = "$(date -d 'yesterday' '+%Y-%m-%d')";
};
};
config = mkOption {
type = types.attr;
description = "The configuration of tauron-scrapper command";
@@ -93,7 +106,7 @@ in {
path = [self.packages.${system}.tauron-scrapper coreutils-full];
script = ''
tauron-scrapper -c ${yamlConfig} -d "$(date -d "yesterday" '+%Y-%m-%d')"
tauron-scrapper -c ${yamlConfig} ${lib.cli.toGNUCommandLineShell {} cfg.cliArgs}
'';
};
};

View File

@@ -12,6 +12,8 @@ export const run = () => {
.version('0.0.1')
.requiredOption('-c, --config <file>', 'sets the path to the YAML config file')
.requiredOption('-d, --date <date>', 'sets the date of measurement intended to be fetched (in YYYY-MM-DD format)')
.requiredOption('-i, --interval <value>', 'sets the time interval between consecutive requests of measurement for time ranges (in ms)', "2000")
.option('-t, --to <date>', 'sets the end date of measurement intended to be fetched (in YYYY-MM-DD format). If provided, the -d option acts as a start date.')
.parse()
.opts<CLIOptions>();
@@ -22,13 +24,28 @@ export const run = () => {
}
const date = dayjs(options.date, 'YYYY-MM-DD');
const to = options.to && dayjs(options.to, 'YYYY-MM-DD');
if (!date.isValid) {
throw new Error(`Invalid date: ${options.date}, expected date to be of 'YYYY-MM-DD' format`);
}
if (to && !to.isValid) {
throw new Error(`Invalid 'to' date: ${options.to}, expected date to be of 'YYYY-MM-DD' format`);
}
const interval = parseInt(options.interval);
const tauron = new Tauron(config.tauron);
const fetcher = new Fetcher(config, tauron);
fetcher.fetch(date);
if (!to) {
fetcher.fetch(date);
return
}
if(to) {
fetcher.fetchRange(date, to, interval);
}
}

View File

@@ -20,7 +20,7 @@ export class Fetcher {
* @param date - the measurement date (note, that 'todays' date may not be available at the time being when request is made)
* @returns the measurement data
*/
async fetch(date: Dayjs): Promise<Measurement> {
async fetch(date: Dayjs): Promise<Measurement> {
const normalizedDate = date.startOf('day');
console.log(`Fetching measurements for: ${normalizedDate.format("YYYY-MM-DD")}`);
@@ -44,8 +44,9 @@ export class Fetcher {
return measurement;
}
async fetchRange(from: Dayjs, to: Dayjs, requestInterval = 1000) {
for(let date = from; date.isBefore(to); date = date.add(1, 'day')) {
async fetchRange(from: Dayjs, to: Dayjs, requestInterval = 1000) {
console.log(`Fetching data from time range: ${from.format("MM.DD.YYYY")} - ${to.format("MM.DD.YYYY")}`)
for(let date = from; date.isBefore(to.add(1, 'day')); date = date.add(1, 'day')) {
this.fetch(date);
await sleep(requestInterval + gaussianRandom(200, 2000));
}

View File

@@ -1,4 +1,6 @@
export type CLIOptions = {
config: string;
date: string;
interval: string;
to?: string;
}