Perf/avoid redundant fetches (#144)

* Only fetch statistics once per entity, even if multiple traces show different ones (e.g max, min, ...)
* Same for multiple attributes of the same entity

Author: dbuezas

src/cache/Cache.ts

@@ -8,9 +8,12 @@ import {
   EntityConfig,
   isEntityIdStateConfig,
   isEntityIdStatisticsConfig,
-  HistoryInRange,
-  EntityState,
+  CachedEntity,
+  CachedStatisticsEntity,
+  CachedStateEntity,
 } from "../types";
+import { groupBy } from "lodash";
+import { StatisticValue } from "../recorder-types";
 
 export function mapValues<T, S>(
   o: Record<string, T>,
@@ -24,7 +27,10 @@ async function fetchSingleRange(
   [startT, endT]: number[],
   significant_changes_only: boolean,
   minimal_response: boolean
-): Promise<HistoryInRange> {
+): Promise<{
+  range: [number, number];
+  history: CachedEntity[];
+}> {
   // We fetch slightly more than requested (i.e the range visible in the screen). The reason is the following:
   // When fetching data in a range `[startT,endT]`, Home Assistant adds a fictitious datapoint at
   // the start of the fetched period containing a copy of the first datapoint that occurred before
@@ -65,7 +71,7 @@ async function fetchSingleRange(
   const start = new Date(startT - 1);
   endT = Math.min(endT, Date.now());
   const end = new Date(endT);
-  let history: EntityState[];
+  let history: CachedEntity[];
   if (isEntityIdStatisticsConfig(entity)) {
     history = await fetchStatistics(hass, entity, [start, end]);
   } else {
@@ -90,9 +96,9 @@ async function fetchSingleRange(
 
 export function getEntityKey(entity: EntityConfig) {
   if (isEntityIdAttrConfig(entity)) {
-    return `${entity.entity}::${entity.attribute}`;
+    return `${entity.entity}::attribute`;
   } else if (isEntityIdStatisticsConfig(entity)) {
-    return `${entity.entity}::statistics::${entity.statistic}::${entity.period}`;
+    return `${entity.entity}::statistics::${entity.period}`;
   } else if (isEntityIdStateConfig(entity)) {
     return entity.entity;
   }
@@ -102,10 +108,10 @@ export function getEntityKey(entity: EntityConfig) {
 const MIN_SAFE_TIMESTAMP = Date.parse("0001-01-02T00:00:00.000Z");
 export default class Cache {
   ranges: Record<string, TimestampRange[]> = {};
-  histories: Record<string, EntityState[]> = {};
+  histories: Record<string, CachedEntity[]> = {};
   busy = Promise.resolve(); // mutex
 
-  add(entity: EntityConfig, states: EntityState[], range: [number, number]) {
+  add(entity: EntityConfig, states: CachedEntity[], range: [number, number]) {
     const entityKey = getEntityKey(entity);
     let h = (this.histories[entityKey] ??= []);
     h.push(...states);
@@ -122,13 +128,33 @@ export default class Cache {
     this.ranges = {};
     this.histories = {};
   }
-  getHistory(entity: EntityConfig) {
+  getHistory(entity: EntityConfig): CachedEntity[] {
     let key = getEntityKey(entity);
     const history = this.histories[key] || [];
-    return history.map((datum) => ({
-      ...datum,
-      timestamp: datum.timestamp + entity.offset,
-    }));
+    if (isEntityIdStatisticsConfig(entity)) {
+      return (history as CachedStatisticsEntity[]).map((entry) => ({
+        ...entry,
+        timestamp: entry.timestamp + entity.offset,
+        value: entry[entity.statistic],
+      }));
+    }
+    if (isEntityIdAttrConfig(entity)) {
+      return (history as CachedStateEntity[]).map((entry) => ({
+        ...entry,
+        timestamp: entry.timestamp + entity.offset,
+        value: entry.attributes[entity.attribute],
+      }));
+    }
+    if (isEntityIdStateConfig(entity)) {
+      return (history as CachedStateEntity[]).map((entry) => ({
+        ...entry,
+        timestamp: entry.timestamp + entity.offset,
+        value: entry.state,
+      }));
+    }
+    throw new Error(
+      `Unrecognised fetch type for ${(entity as EntityConfig).entity}`
+    );
   }
   async update(
     range: TimestampRange,
@@ -137,31 +163,37 @@ export default class Cache {
     significant_changes_only: boolean,
     minimal_response: boolean
   ) {
-    range = range.map((n) => Math.max(MIN_SAFE_TIMESTAMP, n)); // HA API can't handle negative years
     return (this.busy = this.busy
       .catch(() => {})
       .then(async () => {
-        const promises = entities.map(async (entity) => {
-          const entityKey = getEntityKey(entity);
-          this.ranges[entityKey] ??= [];
-          const offsetRange = [
-            range[0] - entity.offset,
-            range[1] - entity.offset,
-          ];
-          const rangesToFetch = subtractRanges(
-            [offsetRange],
-            this.ranges[entityKey]
-          );
-          for (const aRange of rangesToFetch) {
-            const fetchedHistory = await fetchSingleRange(
-              hass,
-              entity,
-              aRange,
-              significant_changes_only,
-              minimal_response
+        range = range.map((n) => Math.max(MIN_SAFE_TIMESTAMP, n)); // HA API can't handle negative years
+        const parallelFetches = Object.values(groupBy(entities, getEntityKey));
+        const promises = parallelFetches.flatMap(async (entityGroup) => {
+          // Each entity in entityGroup will result in exactly the same fetch
+          // But these may differ once the offsets PR is merged
+          // Making these fetches sequentially ensures that the already fetched ranges of each
+          // request are not fetched more than once
+          for (const entity of entityGroup) {
+            const entityKey = getEntityKey(entity);
+            this.ranges[entityKey] ??= [];
+            const offsetRange = [
+              range[0] - entity.offset,
+              range[1] - entity.offset,
+            ];
+            const rangesToFetch = subtractRanges(
+              [offsetRange],
+              this.ranges[entityKey]
             );
-            if (fetchedHistory === null) continue;
-            this.add(entity, fetchedHistory.history, fetchedHistory.range);
+            for (const aRange of rangesToFetch) {
+              const fetchedHistory = await fetchSingleRange(
+                hass,
+                entity,
+                aRange,
+                significant_changes_only,
+                minimal_response
+              );
+              this.add(entity, fetchedHistory.history, fetchedHistory.range);
+            }
           }
         });
 

src/cache/fetch-states.ts

@@ -1,11 +1,10 @@
 import { HomeAssistant } from "custom-card-helpers";
 import {
+  CachedEntity,
   EntityIdAttrConfig,
   EntityIdStateConfig,
-  EntityState,
   HassEntity,
   isEntityIdAttrConfig,
-  isEntityIdStateConfig,
 } from "../types";
 
 async function fetchStates(
@@ -14,16 +13,14 @@ async function fetchStates(
   [start, end]: [Date, Date],
   significant_changes_only?: boolean,
   minimal_response?: boolean
-): Promise<EntityState[]> {
-  const no_attributes_query = isEntityIdStateConfig(entity)
-    ? "no_attributes&"
-    : "";
+): Promise<CachedEntity[]> {
+  const no_attributes_query = isEntityIdAttrConfig(entity)
+    ? ""
+    : "no_attributes&";
   const minimal_response_query =
-    minimal_response && isEntityIdStateConfig(entity)
-      ? "minimal_response&"
-      : "";
+    minimal_response && isEntityIdAttrConfig(entity) ? "" : "minimal_response&";
   const significant_changes_only_query =
-    significant_changes_only && isEntityIdStateConfig(entity) ? "1" : "0";
+    significant_changes_only && isEntityIdAttrConfig(entity) ? "0" : "1";
   const uri =
     `history/period/${start.toISOString()}?` +
     `filter_entity_id=${entity.entity}&` +
@@ -43,14 +40,11 @@ async function fetchStates(
       )}`
     );
   }
-  if (!list) list = [];
-  return list
+  return (list || [])
     .map((entry) => ({
       ...entry,
-      value: isEntityIdAttrConfig(entity)
-        ? entry.attributes[entity.attribute] || null
-        : entry.state,
       timestamp: +new Date(entry.last_updated || entry.last_changed),
+      value: "", // may be state or an attribute. Will be set when getting the history
     }))
     .filter(({ timestamp }) => timestamp);
 }

src/cache/fetch-statistics.ts

@@ -1,12 +1,12 @@
 import { HomeAssistant } from "custom-card-helpers";
 import { Statistics, StatisticValue } from "../recorder-types";
-import { EntityIdStatisticsConfig, EntityState } from "../types";
+import { CachedEntity, EntityIdStatisticsConfig } from "../types";
 
 async function fetchStatistics(
   hass: HomeAssistant,
   entity: EntityIdStatisticsConfig,
   [start, end]: [Date, Date]
-): Promise<EntityState[]> {
+): Promise<CachedEntity[]> {
   let statistics: StatisticValue[] | null = null;
   try {
     const statsP = hass.callWS<Statistics>({
@@ -25,11 +25,12 @@ async function fetchStatistics(
       )}`
     );
   }
-  if (!statistics) statistics = []; //throw new Error(`Error fetching ${entity.entity}`);
-  return statistics.map((entry) => ({
-    ...entry,
-    timestamp: +new Date(entry.start),
-    value: entry[entity.statistic] ?? "",
-  }));
+  return (statistics || [])
+    .map((entry) => ({
+      ...entry,
+      timestamp: +new Date(entry.start),
+      value: "", //depends on the statistic, will be set in getHistory
+    }))
+    .filter(({ timestamp }) => timestamp);
 }
 export default fetchStatistics;

src/plotly-graph-card.ts

@@ -8,7 +8,6 @@ import Plotly from "./plotly";
 import {
   Config,
   EntityConfig,
-  EntityState,
   InputConfig,
   isEntityIdAttrConfig,
   isEntityIdStateConfig,
@@ -208,6 +207,7 @@ export class PlotlyGraph extends HTMLElement {
           } else if (isEntityIdStatisticsConfig(entity)) {
             shouldFetch = true;
           }
+
           if (value !== undefined) {
             this.cache.add(
               entity,
@@ -220,16 +220,14 @@ export class PlotlyGraph extends HTMLElement {
       }
       if (shouldFetch) {
         this.fetch();
-      }
-      if (shouldPlot) {
+      } else if (shouldPlot) {
         this.plot();
       }
     }
     this._hass = hass;
   }
   connectedCallback() {
     this.setupListeners();
-    this.fetch().then(() => (this.contentEl.style.visibility = ""));
   }
   async withoutRelayout(fn: Function) {
     this.isInternalRelayout++;
@@ -741,14 +739,15 @@ export class PlotlyGraph extends HTMLElement {
     if (layout.paper_bgcolor) {
       this.titleEl.style.background = layout.paper_bgcolor as string;
     }
-    await this.withoutRelayout(() =>
-      Plotly.react(
+    await this.withoutRelayout(async () => {
+      await Plotly.react(
         this.contentEl,
         this.getData(),
         layout,
         this.getPlotlyConfig()
-      )
-    );
+      );
+      this.contentEl.style.visibility = "";
+    });
   }
   // The height of your card. Home Assistant uses this to automatically
   // distribute all cards over the available columns.

src/types.ts

@@ -48,7 +48,7 @@ export type EntityConfig = EntityIdConfig & {
   lambda?: (
     y: Datum[],
     x: Date[],
-    raw_entity: EntityState[]
+    raw_entity: CachedEntity[]
   ) => Datum[] | { x?: Datum[]; y?: Datum[] };
   show_value:
     | boolean
@@ -110,15 +110,19 @@ export function isEntityIdStatisticsConfig(
 }
 
 export type Timestamp = number;
-export type EntityState = (HassEntity | StatisticValue) & {
+
+export type CachedStateEntity = HassEntity & {
   fake_boundary_datapoint?: true;
   timestamp: Timestamp;
-  value: number | string;
+  value: number | string | null;
 };
-export type HistoryInRange = {
-  range: [number, number];
-  history: EntityState[];
+export type CachedStatisticsEntity = StatisticValue & {
+  fake_boundary_datapoint?: true;
+  timestamp: Timestamp;
+  value: number | string | null;
 };
+export type CachedEntity = CachedStateEntity | CachedStatisticsEntity;
+
 export type TimestampRange = Timestamp[]; // [Timestamp, Timestamp];
 
 export type HATheme = {