Add support for time ranged requests

module.nix

@@ -16,6 +16,21 @@ in {
   options.services.tauron-scrapper = {
     enable = mkEnableOption "tauron-scrapper";
 
+    cliArgs = mkOption {
+      type = types.attrs;
+      description = "List of CLI arguments. Do not put '-c' argument here as it will be automatically appended basing on the 'config' option.";
+
+      example = {
+        date = "$(date -d '5 days ago' '+%Y-%m-%d')";
+        to = "$(date -d 'yesterday' '+%Y-%m-%d')";
+      };
+
+      default = {
+        date = "$(date -d '5 days ago' '+%Y-%m-%d')";
+        to = "$(date -d 'yesterday' '+%Y-%m-%d')";
+      };
+    };
+
     config = mkOption {
       type = types.attr;
       description = "The configuration of tauron-scrapper command";
@@ -23,8 +38,8 @@ in {
         timezone = "Europe/Warsaw";
 
         tauron = {
-          usernamePath = "/run/tauron.username.key";
+          username = "$_file:/run/tauron.username.key";
-          passwordPath = "/run/tauron.password.key";
+          password = "$_file:/run/tauron.password.key";
           point = "123456";
         };
 
@@ -32,7 +47,7 @@ in {
           influxdb = {
             enable = true;
             databaseURL = "https://influxdb.lan";
-            apiToken = "/run/mqtt/influxdb.token.key";
+            apiToken = "$__file:/run/mqtt/influxdb.token.key";
             organization = "home";
             bucket = "tauron";
           };
@@ -40,8 +55,8 @@ in {
           mqtt = {
             enable = true;
             brokerURL = "https://mqtt.lan";
-            usernamePath = "/run/mqtt.username.key";
+            username = "$__file:/run/mqtt.username.key";
-            passwordPath = "/run/mqtt.password.key";
+            password = "$__file:/run/mqtt.password.key";
             clientId = "tauron-scrapper";
             prefix = "tauron";
             publishOptions = {
@@ -93,7 +108,7 @@ in {
       path = [self.packages.${system}.tauron-scrapper coreutils-full];
 
       script = ''
-        tauron-scrapper -c ${yamlConfig} -d "$(date -d "yesterday" '+%Y-%m-%d')"
+        tauron-scrapper -c ${yamlConfig} ${lib.cli.toGNUCommandLineShell {} cfg.cliArgs}
       '';
     };
   };

src/cli/index.ts

@@ -12,6 +12,8 @@ export const run = () => {
     .version('0.0.1')
     .requiredOption('-c, --config <file>', 'sets the path to the YAML config file')
     .requiredOption('-d, --date <date>', 'sets the date of measurement intended to be fetched (in YYYY-MM-DD format)')
+    .requiredOption('-i, --interval <value>', 'sets the time interval between consecutive requests of measurement for time ranges (in ms)', "2000")
+    .option('-t, --to <date>', 'sets the end date of measurement intended to be fetched (in YYYY-MM-DD format). If provided, the -d option acts as a start date.')
     .parse()
     .opts<CLIOptions>();
 
@@ -22,13 +24,28 @@ export const run = () => {
   }
 
   const date = dayjs(options.date, 'YYYY-MM-DD');
+  const to = options.to && dayjs(options.to, 'YYYY-MM-DD');
   
   if (!date.isValid) {
     throw new Error(`Invalid date: ${options.date}, expected date to be of 'YYYY-MM-DD' format`);
   }
 
+  if (to && !to.isValid) {
+    throw new Error(`Invalid 'to' date: ${options.to}, expected date to be of 'YYYY-MM-DD' format`);
+  }
+
+  const interval = parseInt(options.interval);
+
+
   const tauron = new Tauron(config.tauron);
   const fetcher = new Fetcher(config, tauron);
 
-  fetcher.fetch(date);
+  if (!to) {
+    fetcher.fetch(date);
+    return
+  }
+
+  if(to) {
+    fetcher.fetchRange(date, to, interval);
+  }
 }

src/consumers/influxdb.ts

@@ -3,6 +3,7 @@ import { InfluxDB, Point } from "@influxdata/influxdb-client";
 import { InfluxDBConfig, Measurement } from "@types";
 import { Consumer } from "./abstract";
 import { Dayjs } from "dayjs";
+import { enhancedStringConfig } from "../utils";
 
 export class InfluxDBConsumer extends Consumer<InfluxDBConfig> {
   public name = 'influxdb';
@@ -17,7 +18,7 @@ export class InfluxDBConsumer extends Consumer<InfluxDBConfig> {
   protected async publish({ databaseURL, apiToken, organization, bucket }: InfluxDBConfig, date: Dayjs, measurement: Measurement) {
     const db = new InfluxDB({ 
       url: databaseURL,
-      token: readFileSync(apiToken, 'utf8'),
+      token: enhancedStringConfig(apiToken),
     });
 
     const api = db.getWriteApi(organization, bucket);

src/consumers/mqtt.ts

@@ -4,17 +4,18 @@ import { Measurement } from "@types";
 import { Dayjs } from 'dayjs';
 import { MQTTConfig } from '../types/mqtt';
 import { Consumer } from './abstract';
+import { enhancedStringConfig } from '../utils';
 
 export * from '../types/mqtt';
 
 export class MQTTConsumer extends Consumer<MQTTConfig> {  
   public name = "mqtt";
-  protected requiredFields = ['brokerURL', 'usernamePath', 'passwordPath'] as const;
+  protected requiredFields = ['brokerURL', 'username', 'password'] as const;
 
-  protected publish({ brokerURL, usernamePath, passwordPath, clientId, prefix, publishOptions }: MQTTConfig, date: Dayjs, measurement: Measurement) {    
+  protected publish({ brokerURL, username, password, clientId, prefix, publishOptions }: MQTTConfig, date: Dayjs, measurement: Measurement) {    
     const client = mqtt.connect(brokerURL, {
-      username: fs.readFileSync(usernamePath, 'utf8'),
+      username: enhancedStringConfig(username),
-      password: fs.readFileSync(passwordPath, 'utf8'),
+      password: enhancedStringConfig(password),
       clientId: clientId || "tauron-scrapper"
     });
 

src/fetcher/index.ts

@@ -20,7 +20,7 @@ export class Fetcher {
    * @param date - the measurement date (note, that 'todays' date may not be available at the time being when request is made)
    * @returns the measurement data
    */
-  async fetch(date: Dayjs): Promise<Measurement> {
+  async fetch(date: Dayjs): Promise<Measurement> {  
     const normalizedDate = date.startOf('day');
     console.log(`Fetching measurements for: ${normalizedDate.format("YYYY-MM-DD")}`);
 
@@ -44,8 +44,9 @@ export class Fetcher {
     return measurement;
   }
 
-  async fetchRange(from: Dayjs, to: Dayjs, requestInterval = 1000) {  
+  async fetchRange(from: Dayjs, to: Dayjs, requestInterval = 1000) {
-    for(let date = from; date.isBefore(to); date = date.add(1, 'day')) {
+    console.log(`Fetching data from time range: ${from.format("MM.DD.YYYY")} - ${to.format("MM.DD.YYYY")}`)
+    for(let date = from; date.isBefore(to.add(1, 'day')); date = date.add(1, 'day')) {
       this.fetch(date);
       await sleep(requestInterval + gaussianRandom(200, 2000));
     }

src/tauron/index.ts

@@ -4,6 +4,7 @@ import { EnergyDTO, EnergyRequestDTO, Payload, PowerDTO, PowerRequestDTO, Readin
 import { TauronConfig } from '../config';
 import { Dayjs } from 'dayjs';
 import { withCookies } from './cookie';
+import { enhancedStringConfig } from '../utils';
 
 export * from '../types/tauron';
 
@@ -55,8 +56,8 @@ export class Tauron {
     await this.#http.get(LOGIN_API);
     
     await this.#http.postForm(LOGIN_API, {
-      username: fs.readFileSync(this.#config.usernamePath, 'utf8'),
+      username: enhancedStringConfig(this.#config.username),
-      password: fs.readFileSync(this.#config.passwordPath, 'utf8'),
+      password: enhancedStringConfig(this.#config.password),
       service: BASE_URL
     });
 

src/types/cli.ts

@@ -1,4 +1,6 @@
 export type CLIOptions = {
   config: string;
   date: string;
+  interval: string;
+  to?: string;
 }

src/types/mqtt.ts

@@ -11,12 +11,12 @@ export type MQTTConfig = {
   /**
    * Path to file containing a username of MQTT user.
    */
-  usernamePath: string;
+  username: string;
 
   /**
    * Path to file containing a password of MQTT user.
    */
-  passwordPath: string;
+  password: string;
 
   /**
    * Optional client ID used to connect to MQTT (visible in MQTT broker logs).

src/types/tauron.ts

@@ -2,12 +2,12 @@ export type TauronConfig = {
   /**
    * Path to file containing a username of Tauron account
    */
-  usernamePath: string;
+  username: string;
 
   /**
    * Path to file containing a password of Tauron account
    */
-  passwordPath: string;
+  password: string;
 
   /**
    * The measure point name - should be retrieved with browser dev-tools on website

src/utils/index.ts

@@ -1,3 +1,5 @@
+import { readFileSync } from "fs";
+
 export const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms));
 
 export const gaussianRandom = (mean = 0, stddev = 1) => {
@@ -5,4 +7,20 @@ export const gaussianRandom = (mean = 0, stddev = 1) => {
   let u2 = Math.random();
   let z0 = Math.sqrt(-2.0 * Math.log(u1)) * Math.cos(2.0 * Math.PI * u2);
   return z0 * stddev + mean;
-}
+}
+
+const specialOptions: Record<string, (text: string) => string> = {
+  $__file: (arg: string) => readFileSync(arg, 'utf8').trim()
+};
+
+export const enhancedStringConfig = (value: string) => {  
+  const trimmed = value.trim();
+
+  for(const opt of Object.keys(specialOptions)) {
+    if(trimmed.startsWith(`${opt}:`) && opt in specialOptions) {      
+      return specialOptions[opt](trimmed.slice(opt.length + 1).trim());
+    }
+  }
+
+  return trimmed;
+};