new hcp and scroller implementation

client/src/components/History/CurrentCollection/Panel.vue

@@ -1,15 +1,8 @@
 <!-- When a dataset collection is being viewed, this panel shows the contents of that collection -->
 
 <template>
-    <DscProvider :is-root="isRoot" :collection="selectedCollection" v-slot="{ dsc }">
-        <CollectionContentProvider
-            v-if="dsc"
-            :parent="dsc"
-            v-slot="{
-                payload: { contents = [], startKey = null, topRows = 0, bottomRows = 0 },
-                handlers: { setScrollPos },
-            }"
-        >
+    <DscProvider :is-root="isRoot" :debounce-period="500" :collection="selectedCollection" v-slot="{ dsc }">
+        <CollectionContentProvider v-if="dsc" :parent="dsc" v-slot="{ loading, payload, setScrollPos }">
             <ExpandedItems
                 :scope-key="selectedCollection.id"
                 :get-item-key="(item) => item.type_id"
@@ -25,13 +18,11 @@
                     </template>
 
                     <template v-slot:listing>
+                        <!-- <pre>{{ payload }}</pre> -->
                         <Scroller
                             key-field="element_index"
-                            :item-height="36"
-                            :items="contents"
-                            :scroll-to="startKey"
-                            :top-placeholders="topRows"
-                            :bottom-placeholders="bottomRows"
+                            :class="{ loadingBackground: loading }"
+                            v-bind="payload"
                             @scroll="setScrollPos"
                         >
                             <template v-slot="{ item, index }">

client/src/components/History/CurrentHistoryPanel.vue

@@ -1,5 +1,5 @@
 <template>
-    <CurrentUser v-slot="{ user }">
+    <CurrentUser class="d-flex flex-column" v-slot="{ user }">
         <UserHistories v-if="user" :user="user" v-slot="{ currentHistory, histories, handlers }">
             <HistoryPanel v-if="currentHistory" :history="currentHistory" v-on="handlers">
                 <template v-slot:nav>

client/src/components/History/History.vue

@@ -1,12 +1,11 @@
 <template>
     <HistoryContentProvider
         :parent="history"
-        v-slot="{
-            loading,
-            params,
-            payload: { contents = [], startKey = null, topRows = 0, bottomRows = 0, totalMatches = 0 },
-            handlers: { updateParams, manualReload, setScrollPos },
-        }"
+        :params="params"
+        :disable-poll="false"
+        :debug="false"
+        :debounce-period="500"
+        v-slot="{ loading, payload, manualReload, setScrollPos }"
     >
         <ExpandedItems
             :scope-key="history.id"
@@ -43,32 +42,38 @@
                         <ContentOperations
                             v-if="!history.empty"
                             :history="history"
-                            :total-matches="totalMatches"
+                            :total-matches="payload.totalMatches"
                             :loading="loading"
-                            :params="params"
-                            @update:params="updateParams"
+                            :params.sync="params"
                             :content-selection="selectedItems"
                             @update:content-selection="selectItems"
                             :show-selection="showSelection"
                             @update:show-selection="setShowSelection"
                             @resetSelection="resetSelection"
-                            @selectAllContent="selectItems(contents)"
+                            @selectAllContent="selectItems(payload.contents)"
                             @manualReload="manualReload"
                         />
                     </template>
 
                     <template v-slot:listing>
                         <HistoryEmpty v-if="history.empty" class="m-2" />
-
+                        <!-- <div v-else>
+                            <input
+                                type="range"
+                                min="0.0"
+                                max="1.0"
+                                step="0.1"
+                                :value="scrollPos.cursor"
+                                @input="setScrollPos({ cursor: 1.0 * $event.srcElement.value })"
+                            />
+                            <pre>{{ payload | reportPayload }}</pre>
+                        </div> -->
                         <Scroller
                             v-else
                             :class="{ loadingBackground: loading }"
                             key-field="hid"
-                            :item-height="36"
-                            :items="contents"
-                            :scroll-to="startKey"
-                            :top-placeholders="topRows"
-                            :bottom-placeholders="bottomRows"
+                            v-bind="payload"
+                            :debug="false"
                             @scroll="setScrollPos"
                         >
                             <template v-slot="{ item, index }">
@@ -96,7 +101,7 @@
 </template>
 
 <script>
-import { History } from "./model";
+import { History, SearchParams } from "./model";
 import { HistoryContentProvider, ExpandedItems, SelectedItems } from "./providers";
 import Layout from "./Layout";
 import HistoryMessages from "./HistoryMessages";
@@ -107,7 +112,16 @@ import ToolHelpModal from "./ToolHelpModal";
 import Scroller from "./Scroller";
 import { HistoryContentItem } from "./ContentItem";
 
+const reportPayload = (p) => {
+    const { contents = [], ...theRest } = p;
+    const hids = contents.map((o) => o.hid);
+    return { hids, ...theRest };
+};
+
 export default {
+    filters: {
+        reportPayload,
+    },
     components: {
         HistoryContentProvider,
         Layout,
@@ -124,6 +138,12 @@ export default {
     props: {
         history: { type: History, required: true },
     },
+    data() {
+        return {
+            params: new SearchParams(),
+            useItemSelection: false,
+        };
+    },
     computed: {
         historyId() {
             return this.history.id;

client/src/components/History/Scroller.vue

@@ -0,0 +1,325 @@
+<template>
+    <div ref="container" class="scrollContainer" @wheel.prevent="onWheel">
+        <!-- mousewheel moves up and down linearly, allows for fine-tuning of view by moving just a
+        few rows at a time -->
+        <div ref="listing" class="listing">
+            <ul>
+                <li v-for="(item, index) in itemWindow" :key="getItemKey(item, index)">
+                    <slot :item="item" :index="index" :row-key="getItemKey(item, index)">
+                        <pre>{{ item }}</pre>
+                    </slot>
+                </li>
+            </ul>
+        </div>
+
+        <!-- big blank scroll region uses scroll bar to drag view to entirely new region, calculated
+        by visual height of the scroller knob --->
+        <div v-show="showScroller" ref="scrollSliderContainer" class="scrollSliderContainer" @scroll="onScroll">
+            <div ref="scrollSlider"></div>
+        </div>
+    </div>
+</template>
+
+<script>
+import { debounce } from "lodash";
+import { SearchParams } from "./model";
+
+const clean = (o) => JSON.parse(JSON.stringify(o));
+
+export default {
+    props: {
+        // field in the item data that represents the "primary" key, less for caching purposes than
+        // Vue re-rendering comparison purposes. Should be a unique value from the list of data, and
+        // is typically HID for history contents and element_index for collection contents
+        keyField: { type: String, required: true },
+
+        // A list of items to render in the immediate region of the scroll pos
+        contents: { type: Array, required: true },
+
+        // index from contents to start rendering, will be 0 for top of list,
+        // we provide more contents than are currently rendered so the user can quickly
+        // scroll outside the immediate window and still have a little data
+        startKeyIndex: { type: Number, default: 0 },
+
+        // known total matches for this list of content independent of pagination or
+        // view windowing concerns, the raw data row count
+        totalMatches: { type: Number, default: 0 },
+
+        // number of undelivered rows above contents
+        topRows: { type: Number, default: 0 },
+        bottomRows: { type: Number, default: 0 },
+
+        // Number of items from contents to render starting from the startKeyIndex
+        pageSize: { type: Number, default: SearchParams.pageSize },
+
+        // avoid an infinite loop when we visually adjust the scroller bar upon
+        // receiving new content by suppressing outgoing update events until the
+        // DOM update has finished.
+        suppressionPeriod: { type: Number, default: 250 },
+
+        // debug flag
+        debug: { type: Boolean, default: false },
+    },
+
+    data() {
+        return {
+            // manual change of start key index, happens when user moves mouse-wheel
+            manualStartIndex: null,
+
+            // show/hide the draggable scrollbar
+            showScroller: false,
+        };
+    },
+
+    computed: {
+        // The index to start rendering content at, can be manually adjusted with
+        // the mouse wheel, and must be updated when new contents come in
+        itemStartIndex() {
+            return this.manualStartIndex ?? this.startKeyIndex;
+        },
+
+        // The slice of contents to render right now, this is the actual data we loop over in the template
+        itemWindow() {
+            return this.contents.slice(this.itemStartIndex, this.itemStartIndex + this.pageSize);
+        },
+
+        // starting row index if we include the undelivered topRows, this is a
+        // number representing how far down we are in the complete data set
+        dataStartIndex() {
+            return this.itemStartIndex + this.topRows;
+        },
+
+        // The portion of the way down the total list as descrbed by the input props
+        cursor() {
+            return this.totalMatches > 0 ? this.dataStartIndex / this.totalMatches : 0;
+        },
+
+        // user switched key, this is the row that was switched to
+        manualStartKey() {
+            const newIdx = this.manualStartIndex;
+            return this.contents?.[newIdx]?.[this.keyField];
+        },
+    },
+
+    watch: {
+        // when contents are updated, reset manualStartIndex
+        // to pay attention to the passed in startKeyIndex instead
+        // of the manual value we set when the user moved the mouse wheel
+        contents() {
+            this.manualStartIndex = null;
+        },
+
+        // When user moves the mouse wheel request more data by row key, since
+        // we should have that value
+        manualStartKey(key, oldKey) {
+            if (key !== undefined && key !== oldKey) {
+                this.$emit("scroll", { key });
+            }
+        },
+
+        cursor(newVal) {
+            this.debouncedAdjustScrollTop(newVal);
+        },
+    },
+
+    created() {
+        // non-reactive data
+        this.suppressEvents = false;
+        this.suppressionTimeout = null;
+        this.debouncedUpdateCursor = debounce(this.updateCursor, 100);
+        this.debouncedAdjustScrollTop = debounce(this.adjustScrollTop, 100);
+    },
+
+    updated() {
+        this.checkScrollbars();
+        // this.reportProps("updated");
+    },
+
+    mounted() {
+        this.checkScrollbars();
+    },
+
+    methods: {
+        log(...args) {
+            if (this.debug) {
+                console.log("Scroller ->", ...args);
+            }
+        },
+
+        reportProps(label) {
+            const data = clean(this._data);
+            const props = clean(this._props);
+            console.group(label);
+            console.dir({ data, props });
+            console.groupEnd();
+        },
+
+        getItemKey(item) {
+            return item[this.keyField];
+        },
+
+        // move the list up and down with the mouse wheel incrementally
+        // shifts manual startIndex up and down
+
+        onWheel({ deltaY }) {
+            // no wheel if list too short to warrant a scroll bar
+            // let the browser figure that out
+            if (this.showScroller) {
+                if (deltaY > 0) this.wheelDown();
+                if (deltaY < 0) this.wheelUp();
+            }
+        },
+
+        wheelDown(n = 1) {
+            this.manualStartIndex = Math.min(this.itemStartIndex + n, this.totalMatches - 1);
+        },
+
+        wheelUp(n = 1) {
+            this.manualStartIndex = Math.max(0, this.itemStartIndex - n);
+        },
+
+        // move to whole new regions with the scrollbar, takes the percentage
+        // of the height of the fake placeholder box and turns that into a 0-1
+        // value which we send out on scrollPos
+
+        onScroll() {
+            this.debouncedUpdateCursor();
+        },
+
+        // determines cursor from scrollTop, need to wait until after render to calculate
+        // this stuff because it depends on DOM elements
+        updateCursor() {
+            if (this.suppressEvents) return;
+
+            const box = this.$refs.scrollSliderContainer;
+            const slider = this.$refs.scrollSlider;
+
+            this.log("updateCursor", slider.clientHeight, box.clientHeight, box.scrollTop);
+
+            const maxHeight = Math.abs(slider.clientHeight - box.clientHeight);
+            const cursor = maxHeight > 0 ? box.scrollTop / maxHeight : 0;
+            this.$emit("scroll", { cursor });
+        },
+
+        // adjust scrolltop from scrollPos prop, happens on a content load
+        // when the content window changes, may need to adjust the position
+        // of the scroll bar handle, but we don't want to retrigger a bunch
+        // of extraneous scroll events in the process.
+
+        adjustScrollTop(newCursor) {
+            const box = this.$refs.scrollSliderContainer;
+            const slider = this.$refs.scrollSlider;
+            if (!(box && slider)) return; // might be empty history panel
+
+            const maxHeight = slider.clientHeight - box.clientHeight;
+
+            // new scrolltop represented by new cursor
+            // scrolltop should represent the height of the first displayed
+            // row of the content in relation to the total number of rows in
+            // the result set
+            const newScrollTop = 1.0 * maxHeight * newCursor;
+
+            // existing scrolltop
+            const existingScrollTop = box.scrollTop;
+
+            // this is how far we need to adjust
+            // If we're off by more than fudge, schedule an event-free adjustment
+            // to make it visually look right
+            const fudge = this.getFirstRowHeight();
+            const adjustment = Math.abs(newScrollTop - existingScrollTop);
+            if (adjustment > fudge) {
+                this.log("adjustScrollTop", newCursor, newScrollTop, adjustment);
+                this.setScrollTop(newScrollTop);
+            }
+        },
+
+        getFirstRowHeight() {
+            const rowHeight = this.$refs?.listing.querySelector("ul > li")?.offsetHeight || 10;
+            return parseInt(rowHeight);
+        },
+
+        setScrollTop(newScrollTop) {
+            this.suppressEvents = true;
+            if (this.suppressionTimeout) {
+                clearTimeout(this.suppressionTimeout);
+                this.suppressionTimeout = null;
+            }
+
+            this.log("setScrollTop", newScrollTop, this.suppressionPeriod);
+
+            const box = this.$refs.scrollSliderContainer;
+            box.scrollTop = Math.floor(newScrollTop);
+
+            this.suppressionTimeout = setTimeout(() => {
+                this.log("unsupressing");
+                this.suppressEvents = false;
+            }, this.suppressionPeriod);
+        },
+
+        // Determine whether the scrollbar is showing on the actual list,
+        // this value is used to toggle the overlay scroller on/off
+
+        checkScrollbars() {
+            const underScrollWidth = this.scrollBarWidth("listing");
+            this.showScroller = underScrollWidth > 0 || this.topRows > 0;
+        },
+
+        scrollBarWidth(refName) {
+            const listing = this.$refs?.[refName];
+            if (!listing) return 0;
+            // offset width includes scrollbar, clientWidth does not
+            return listing.offsetWidth - listing.clientWidth;
+        },
+    },
+};
+</script>
+
+<style lang="scss">
+.scrollContainer {
+    position: absolute;
+    top: 0;
+    left: 0;
+    right: 0;
+    bottom: 0;
+
+    .listing,
+    .scrollSliderContainer {
+        position: absolute;
+        top: 0;
+        left: 0;
+        right: 0;
+        bottom: 0;
+    }
+
+    .listing {
+        overflow-y: scroll;
+        z-index: 0;
+
+        &::-webkit-scrollbar,
+        &::-webkit-scrollbar-track,
+        &::-webkit-scrollbar-thumb {
+            background-color: transparent;
+        }
+    }
+
+    .scrollSliderContainer {
+        left: unset;
+        width: 100px;
+        overflow-y: scroll;
+        z-index: 1;
+        > div {
+            height: 5000px;
+        }
+    }
+
+    ul {
+        list-style: none;
+    }
+
+    ul,
+    li {
+        padding: 0;
+        margin: 0;
+    }
+}
+</style>

client/src/components/History/Scroller/Resize.js

@@ -1,13 +0,0 @@
-/* a very simple directive that alerts parent when its height changes */
-
-const doUpdate = (label) => (el, binding) => {
-    const handler = binding.value;
-    handler(el.clientHeight, binding.modifiers);
-};
-
-export const Resize = {
-    bind: doUpdate("bind"),
-    componentUpdated: doUpdate("componentUpdated"),
-};
-
-export default Resize;

client/src/components/History/Scroller/Scroll.js

@@ -1,37 +0,0 @@
-function inserted(el, binding) {
-    const { self = false } = binding.modifiers || {};
-    const value = binding.value;
-
-    const options = (typeof value === "object" && value.options) || {
-        passive: true,
-    };
-
-    const handler = typeof value === "function" || "handleEvent" in value ? value : value.handler;
-
-    const target = self ? el : binding.arg ? document.querySelector(binding.arg) : window;
-
-    if (!target) return;
-
-    target.addEventListener("scroll", handler, options);
-
-    el._onScroll = {
-        handler,
-        options,
-        // Don't reference self
-        target: self ? undefined : target,
-    };
-}
-
-function unbind(el) {
-    if (!el._onScroll) return;
-    const { handler, target = el } = el._onScroll;
-    target.removeEventListener("scroll", handler);
-    delete el._onScroll;
-}
-
-export const Scroll = {
-    inserted,
-    unbind,
-};
-
-export default Scroll;

client/src/components/History/Scroller/Scroller.js

@@ -1,239 +0,0 @@
-import inViewport from "in-viewport";
-import Scroll from "./Scroll";
-import { cssLength, notNegative } from "./util";
-import "./scroller.scss";
-
-export default {
-    directives: { Scroll },
-
-    props: {
-        keyField: { type: String, required: true },
-        scrollDebounce: { type: Number, required: false, default: 250 },
-        itemHeight: { required: true, validator: notNegative },
-        items: { type: Array, required: true },
-        topPlaceholders: { type: Number, default: 0 },
-        bottomPlaceholders: { type: Number, default: 0 },
-        suppressionPeriod: { type: Number, default: 250 },
-        scrollTo: { default: null },
-    },
-
-    data() {
-        return {
-            scrollTop: 0,
-        };
-    },
-
-    computed: {
-        contentStyle() {
-            return {
-                paddingTop: cssLength(this.topPlaceholders * this.itemHeight),
-                paddingBottom: cssLength(this.bottomPlaceholders * this.itemHeight),
-            };
-        },
-
-        cursor() {
-            if (this.scrollTop > 0) {
-                const height = this.getRefProp("content", "clientHeight", 0);
-                if (undefined !== height && height > 0) {
-                    return (1.0 * this.scrollTop) / height;
-                }
-            }
-            return 0;
-        },
-
-        scrollEventPayload() {
-            const el = this.getFirstVisibleElement();
-            return {
-                cursor: this.cursor,
-                key: el?.dataset.key,
-                keyIndex: el?.dataset.index,
-            };
-        },
-    },
-
-    watch: {
-        scrollEventPayload(payload) {
-            if (!this._suppressScrollEvent) {
-                this.$emit("scroll", payload);
-            }
-        },
-
-        scrollTo(val) {
-            this.suppress();
-            this.$nextTick(() => {
-                const { scrollToMe, container } = this.$refs;
-                const doScroll =
-                    scrollToMe && val && scrollToMe.dataset.key == val && !inViewport(scrollToMe, { container });
-                if (doScroll) {
-                    scrollToMe.scrollIntoView(true);
-                }
-                setTimeout(() => this.unsuppress(), 100);
-            });
-        },
-    },
-
-    methods: {
-        onScroll() {
-            this.scrollTop = this.$refs.container.scrollTop;
-        },
-
-        //#region Dom Utils
-
-        getScrollTop(refName = "container") {
-            return this.getRefProp(refName, "scrollTop");
-        },
-
-        getRefProp(refName, propName, defaultVal = undefined) {
-            if (refName in this.$refs) {
-                if (propName in this.$refs[refName]) {
-                    return this.$refs[refName][propName];
-                }
-            }
-            return defaultVal;
-        },
-
-        getRowElementByKey(key) {
-            if (this.$refs.contentlist) {
-                const list = Array.from(this.$refs.contentlist.children);
-                return list.find((el) => el.dataset.key == key);
-            }
-            return undefined;
-        },
-
-        getFirstVisibleElement() {
-            const { container, contentlist } = this.$refs;
-            let result = undefined;
-            if (container && contentlist) {
-                for (const row of contentlist.children) {
-                    if (inViewport(row, { container })) {
-                        result = row;
-                        break;
-                    }
-                }
-            }
-            return result;
-        },
-
-        //#endregion
-
-        //#region Event suppression
-
-        suppress() {
-            this._suppressScrollEvent = true;
-        },
-
-        unsuppress() {
-            if (this._suppressTimeoutId) {
-                clearTimeout(this._suppressTimeoutId);
-                this._suppressTimeoutId = null;
-            }
-            this._suppressTimeoutId = setTimeout(() => {
-                this._suppressScrollEvent = false;
-            }, this.suppressionPeriod);
-        },
-
-        //#endregion
-
-        //#region Rendering
-
-        renderList() {
-            return this.items.map(this.renderItem);
-        },
-
-        renderItem(item, index) {
-            const h = this.$createElement;
-            const key = item[this.keyField];
-            const slotChild = this.renderSlot("default", { key, index, item });
-
-            let ref = undefined;
-            let staticClass = "";
-            if (key == this.scrollTo) {
-                ref = "scrollToMe";
-                staticClass = "scrollToMe";
-            }
-
-            const attrs = {
-                // index considering the missing data rows reprsented by topPlaceholders
-                "data-index": index + this.topPlaceholders,
-                // index with respect to the passed items array
-                "item-index": index,
-                "data-key": key,
-            };
-
-            return h("li", { key, ref, staticClass, attrs }, slotChild);
-        },
-
-        renderSlot(name = "default", data, optional = false) {
-            if (this.$scopedSlots[name]) {
-                return this.$scopedSlots[name](data instanceof Function ? data() : data);
-            } else if (this.$slots[name] && (!data || optional)) {
-                return this.$slots[name];
-            }
-            return undefined;
-        },
-
-        //#endregion
-
-        //#region Debugging
-
-        report(label) {
-            console.groupCollapsed(label);
-            console.log("scrollTo", this.scrollTo);
-            console.log("items.length", this.items.length);
-            if (this.items.length > 0) {
-                const list = this.items;
-                console.log("items", `${list[0].hid} -> ${list[list.length - 1].hid}`);
-            }
-            console.log("topPlaceholders", this.topPlaceholders);
-            console.log("bottomPlaceholders", this.bottomPlaceholders);
-            console.groupEnd();
-        },
-
-        //#endregion
-    },
-
-    render(h) {
-        // content list
-        const list = h(
-            "ul",
-            {
-                ref: "contentlist",
-            },
-            this.renderList()
-        );
-
-        // contains all the content + padding
-        const content = h(
-            "div",
-            {
-                ref: "content",
-                staticClass: "scrollContent",
-                style: this.contentStyle,
-                attrs: {
-                    "data-top-rows": this.topPlaceholders,
-                    "data-bottom-rows": this.bottomPlaceholders,
-                },
-            },
-            [list]
-        );
-
-        // outer container, scrollTop lives here
-        const container = h(
-            "div",
-            {
-                ref: "container",
-                staticClass: "historyScroller",
-                directives: [
-                    {
-                        name: "scroll",
-                        modifiers: { self: true },
-                        value: this.onScroll,
-                    },
-                ],
-            },
-            [content]
-        );
-
-        return container;
-    },
-};

client/src/components/History/Scroller/debugUpdate.js

@@ -1,97 +0,0 @@
-/**
- * Debugging tool for figuring out what changed to force Vue to re-render
- */
-
-import diff from "deep-diff";
-
-const Monitor = (vm) => {
-    const collect = (fields) => fields.reduce((acc, fName) => ({ ...acc, [fName]: vm[fName] }), {});
-
-    const verbs = { N: "Added", D: "Removed", E: "Edited", A: "Array Element" };
-
-    function reportDifferences(label, diffs = []) {
-        console.groupCollapsed(label, diffs.length);
-        for (const d of diffs) {
-            try {
-                const { kind, path, index, lhs, rhs, item } = d;
-                const prop = path.join(".");
-                if (kind == "A") {
-                    console.log(verbs[kind], `${prop}[${index}]`, item.rhs);
-                } else {
-                    console.log(verbs[kind], prop, lhs, rhs);
-                }
-            } catch (err) {
-                console.log("oops", err, d);
-            }
-        }
-        console.groupEnd();
-    }
-
-    return {
-        // what are we watching?
-        props: false,
-        data: false,
-        computed: false,
-        get active() {
-            return this.props || this.data || this.computed;
-        },
-
-        // last known values
-        lastStuff: {},
-
-        // changes
-        differences: undefined,
-
-        // set of property names to not bother tracking, handy for large
-        // arrays where we don't want every little change
-        ignoreProps: new Set(),
-
-        getVals() {
-            const computedVals = this.computed ? collect(Object.keys(vm._computedWatchers)) : {};
-            const dataVals = this.data ? collect(Object.keys(vm._data)) : {};
-            const propVals = this.props ? collect(Object.keys(vm._props)) : {};
-            const vals = { ...propVals, ...dataVals, ...computedVals };
-            for (const propName of this.ignoreProps) {
-                delete vals[propName];
-            }
-            return vals;
-        },
-
-        diff() {
-            if (!this.active) return;
-            this.differences = diff(this.lastStuff || {}, this.getVals());
-        },
-
-        store() {
-            if (!this.active) return;
-            this.lastStuff = this.getVals();
-        },
-
-        report() {
-            if (!this.active) return;
-            if (this.differences) {
-                reportDifferences("updated", this.differences);
-                this.differences = undefined;
-            }
-        },
-
-        ignore(field) {
-            this.ignoreProps.add(field);
-        },
-    };
-};
-
-export default {
-    beforeCreate() {
-        this._monitor = Monitor(this);
-    },
-
-    beforeUpdate() {
-        this._monitor.diff();
-    },
-
-    updated() {
-        this._monitor.store();
-        this._monitor.report();
-    },
-};

client/src/components/History/Scroller/index.js

@@ -1,8 +0,0 @@
-/**
- * Had to build a custom virtual scroller to suit our needs.
- */
-
-import Scroller from "./Scroller";
-
-export { Scroller };
-export default Scroller;

client/src/components/History/Scroller/scroller.scss

@@ -1,30 +0,0 @@
-.historyScroller {
-    position: absolute;
-    width: 100%;
-    height: 100%;
-    overflow: auto;
-}
-.historyScroller {
-    ul {
-        list-style: none;
-    }
-
-    ul,
-    li {
-        padding: 0;
-        margin: 0;
-    }
-
-    li.bench {
-        overflow: hidden;
-    }
-
-    .scrollContent {
-        transition: padding-top 0.5s;
-        transition: padding-bottom 0.5s;
-    }
-
-    // .scrollToMe > .content-item {
-    //     background-color: orange !important;
-    // }
-}

client/src/components/History/Scroller/util.js

@@ -1,19 +0,0 @@
-// dumb formatter
-export function cssLength(str, unit = "px") {
-    if (str == null || str === "") {
-        return undefined;
-    } else if (isNaN(+str)) {
-        return String(str);
-    } else {
-        const val = Number(str);
-        return `${val}${unit}`;
-    }
-}
-
-// validator
-export function notNegative(rawVal) {
-    const val = Number(rawVal);
-    if (isNaN(val)) return false;
-    if (val < 0) return false;
-    return true;
-}

client/src/components/History/Scroller/watchStuff.js

@@ -1,59 +0,0 @@
-const watchChangeHandler = (label) => (newVal, oldVal) => {
-    if (newVal != oldVal) {
-        console.log(label, newVal);
-    }
-};
-
-export default {
-    watch: {
-        items: {
-            immediate: true,
-            handler: watchChangeHandler("PROP: items"),
-        },
-
-        itemHeight: {
-            immediate: true,
-            handler: watchChangeHandler("PROP: itemHeight"),
-        },
-
-        scrollStartKey: {
-            immediate: true,
-            handler: watchChangeHandler("PROP: scrollStartKey"),
-        },
-
-        topPlaceholders: {
-            immediate: true,
-            handler: watchChangeHandler("PROP: topPlaceholders"),
-        },
-
-        bottomPlaceholders: {
-            immediate: true,
-            handler: watchChangeHandler("PROP: bottomPlaceholders"),
-        },
-
-        scrollTop: {
-            immediate: true,
-            handler: watchChangeHandler("DATA: scrollTop"),
-        },
-
-        scrollerHeight: {
-            immediate: true,
-            handler: watchChangeHandler("DATA: scrollerHeight"),
-        },
-
-        topPlaceholderHeight: {
-            immediate: true,
-            handler: watchChangeHandler("COMPUTED: topPlaceholderHeight"),
-        },
-
-        bottomPlaceholderHeight: {
-            immediate: true,
-            handler: watchChangeHandler("COMPUTED: bottomPlaceholderHeight"),
-        },
-
-        contentHeight: {
-            immediate: true,
-            handler: watchChangeHandler("DATA: contentHeight"),
-        },
-    },
-};

client/src/components/History/caching/CacheApi.js

@@ -8,7 +8,6 @@
  * instances.
  */
 
-import { pipe } from "rxjs";
 import { content$, dscContent$ } from "./db/observables";
 import { monitorQuery } from "./db/monitorQuery";
 
@@ -16,17 +15,15 @@ export * from "./db/promises";
 
 export { loadDscContent } from "./loadDscContent";
 export { loadHistoryContents, clearHistoryDateStore } from "./loadHistoryContents";
-export { monitorHistoryContent } from "./monitorHistoryContent";
+export { monitorHistoryContent, monitorCollectionContent } from "./monitorHistoryContent";
 export { wipeDatabase } from "./db/wipeDatabase";
 
 // generic content query monitor
 export const monitorContentQuery = (cfg = {}) => {
-    const monitorCfg = { db$: content$, ...cfg };
-    return pipe(monitorQuery(monitorCfg));
+    return monitorQuery({ db$: content$, ...cfg });
 };
 
 // generic collection content monitor
 export const monitorDscQuery = (cfg = {}) => {
-    const monitorCfg = { db$: dscContent$, ...cfg };
-    return pipe(monitorQuery(monitorCfg));
+    return monitorQuery({ db$: dscContent$, ...cfg });
 };

client/src/components/History/caching/CacheWorker.worker.js

@@ -13,6 +13,7 @@ const {
     monitorContentQuery,
     monitorDscQuery,
     monitorHistoryContent,
+    monitorCollectionContent,
     loadHistoryContents,
     loadDscContent,
     pollHistory,
@@ -22,13 +23,15 @@ const {
 expose({
     configure,
 
-    // observables
+    // observable operators
     monitorContentQuery: asObservable(monitorContentQuery),
     monitorDscQuery: asObservable(monitorDscQuery),
     monitorHistoryContent: asObservable(monitorHistoryContent),
+    monitorCollectionContent: asObservable(monitorCollectionContent),
     loadHistoryContents: asObservable(loadHistoryContents),
     loadDscContent: asObservable(loadDscContent),
     pollHistory: asObservable(pollHistory),
 
+    // promise functions
     ...promises,
 });

client/src/components/History/caching/__mocks__/index.js

@@ -1,19 +1,10 @@
-export {
-    // observables
-    monitorContentQuery,
-    monitorDscQuery,
-    monitorHistoryContent,
-    loadHistoryContents,
-    loadDscContent,
-    pollHistory,
-    // promises
-    cacheContent,
-    getCachedContent,
-    bulkCacheContent,
-    uncacheContent,
-    cacheCollectionContent,
-    getCachedCollectionContent,
-    bulkCacheDscContent,
-    // utility & debugging
-    wipeDatabase,
-} from "../CacheApi";
+// For testing, bypass the worker convrsions
+// and wrap each function in a jest mock wrapper
+
+const cacheapi = require("../CacheApi");
+
+const mockedModule = Object.entries(cacheapi).reduce((mod, [fnName, orig]) => {
+    return { ...mod, [fnName]: jest.fn(orig) };
+}, {});
+
+module.exports = mockedModule;

client/src/components/History/caching/db/find.js

@@ -1,6 +1,6 @@
 import { pipe } from "rxjs";
 import { mergeMap } from "rxjs/operators";
-import { needs } from "../operators/needs";
+import { show, needs } from "utils/observable";
 
 /**
  * Pouchdb-find as an operator
@@ -8,8 +8,11 @@ import { needs } from "../operators/needs";
  *
  * @param {Observable} db$ Observable pouchDb instance
  */
-export const find = (db$) => {
+export const find = (db$, cfg = {}) => {
+    const { label = "find", debug = false } = cfg;
+
     return pipe(
+        show(debug, (request) => console.log(`${label} -> request`, request)),
         needs(db$),
         mergeMap(async (inputs) => {
             const [request, db] = inputs;
@@ -33,6 +36,7 @@ export const find = (db$) => {
             }
 
             return docs;
-        })
+        }),
+        show(debug, (result) => console.log(`${label} -> result`, result))
     );
 };

client/src/components/History/caching/db/monitorQuery.js

@@ -1,19 +1,22 @@
 import deepEqual from "deep-equal";
-import { isObservable, partition, merge, concat, of, pipe } from "rxjs";
-import { tap, map, switchMap, debounceTime, distinctUntilChanged, pluck, filter } from "rxjs/operators";
+import { isObservable, concat, of } from "rxjs";
+import { concatAll, map, switchMap, debounceTime, distinctUntilChanged, pluck, filter } from "rxjs/operators";
 import { matchesSelector } from "pouchdb-selector-core";
 import { find } from "./find";
 import { changes } from "./changes";
+import { show } from "utils/observable";
 
 /**
- * Turns a selector into a live cache result emitter.
+ * Turns a selector into a live cache result emitter. Emits objects with
+ * a document object and a match boolean flag ({ doc, match: true }) indicating
+ * whether or not this document matches the request$ selector
  *
  * @param {Observable} db$ Observable PouchDB instance
  * @param {Observable} request$ Observable Pouchdb-find configuration
  */
 // prettier-ignore
 export const monitorQuery = (cfg = {}) => (request$) => {
-    const { db$ = null, inputDebounce = 0 } = cfg;
+    const { db$, inputDebounce = 0, debug = false, label } = cfg;
 
     if (!isObservable(db$)) {
         throw new Error("Please pass a pouch database observable to monitorQuery");
@@ -27,84 +30,27 @@ export const monitorQuery = (cfg = {}) => (request$) => {
     return debouncedRequest$.pipe(
         switchMap((request) => {
             const { selector } = request;
-            const idSet = new Set();
+            const docMatch = doc => matchesSelector(doc, selector);
 
             // do a search of the cache first
             const initial$ = of(request).pipe(
-                find(db$),
-                tap((docs) => docs.forEach((doc) => idSet.add(doc._id))),
-                map((docs) => ({ action: ACTIONS.INITIAL, initialMatches: docs })),
+                find(db$, { debug, label }),
+                concatAll(),
+                map((doc) => ({ doc, match: true, initial: true })),
+                show(debug, (change) => console.log("monitorQuery: initial", change)),
             );
 
             // later changes
-            const changedDocs$ = db$.pipe(
-                changes(),
-                filter(update => update.doc !== undefined),
+            const updates$ = db$.pipe(
+                changes({ debug, label }),
                 pluck("doc"),
-            );
-
-            // split stream into insert/update/remove
-            const docMatch = doc => matchesSelector(doc, selector);
-            const [ inSet$, notInSet$ ] = partition(changedDocs$, doc => idSet.has(doc._id));
-            const [ updateEvt$, removeEvt$ ] = partition(inSet$, docMatch);
-            const insertEvt$ = notInSet$.pipe(filter(docMatch));
-
-            // merge all updates
-            const updates$ = merge(
-                updateEvt$.pipe(
-                    tap(doc => idSet.add(doc._id)),
-                    map(doc => ({ action: ACTIONS.UPDATE, doc }))
-                ),
-                removeEvt$.pipe(
-                    tap(doc => idSet.delete(doc._id)),
-                    map(doc => ({ action: ACTIONS.REMOVE, doc }))
-                ),
-                insertEvt$.pipe(
-                    tap(doc => idSet.add(doc._id)),
-                    map(doc => ({ action: ACTIONS.ADD, doc }))
-                ),
+                filter(Boolean),
+                map((doc) => ({ doc, match: docMatch(doc), update: true })),
+                show(debug, (change) => console.log("monitorQuery: update", change)),
             );
 
             return concat(initial$, updates$);
-        })
+        }),
+        show(debug, (change) => console.log("monitorQuery", change)),
     );
 };
-
-// prettier-ignore
-export const singleUpdateResult = () => {
-    return pipe(
-        map(update => {
-            const { action, initialMatches = [], doc = null } = update;
-
-            // if no match, result is undefined
-            let result = null;
-
-            switch (action) {
-                case ACTIONS.INITIAL:
-                    if (initialMatches.length) {
-                        result = initialMatches[0];
-                    }
-                    break;
-                case ACTIONS.REMOVE:
-                    result = null;
-                    break;
-                case ACTIONS.UPDATE:
-                case ACTIONS.ADD:
-                    if (doc) {
-                        result = doc;
-                    }
-                    break;
-            }
-
-            return result;
-        })
-    )
-};
-
-export const ACTIONS = {
-    INITIAL: "INITIAL",
-    ADD: "ADD",
-    UPDATE: "UPDATE",
-    REMOVE: "REMOVE",
-    IGNORE: "IGNORE",
-};

client/src/components/History/caching/db/monitorQuery.test.js

@@ -1,184 +1,209 @@
 import { of, timer } from "rxjs";
-import { take, takeUntil } from "rxjs/operators";
+import { takeUntil } from "rxjs/operators";
 import { ObserverSpy } from "@hirez_io/observer-spy";
 import { wait } from "jest/helpers";
 import { wipeDatabase } from "./wipeDatabase";
 
-import { monitorQuery, ACTIONS } from "./monitorQuery";
+import { monitorQuery } from "./monitorQuery";
 import { content$, dscContent$ } from "./observables";
-import {
-    cacheContent,
-    getCachedContent,
-    bulkCacheContent,
-    bulkCacheDscContent,
-    cacheCollectionContent,
-    getCachedCollectionContent,
-} from "./promises";
+import { cacheContent, cacheCollectionContent, bulkCacheContent, bulkCacheDscContent } from "./promises";
 
 // test data
 import historyContent from "../../test/json/historyContent.json";
 import collectionContent from "../../test/json/collectionContent.json";
 
-const monitorSpinUp = 100;
+jest.mock("app");
+jest.mock("../../caching");
 
-afterEach(async () => await wipeDatabase());
+const monitorSpinUp = 100;
+const monitorSafetyTimeout = 800;
+
+const pluckAll = (arr, prop) => arr.map((o) => o[prop]);
+
+afterEach(wipeDatabase);
 
 // prettier-ignore
 describe("monitorQuery: history content", () => {
+
     describe("initial results", () => {
-        const selector = { hid: { $gt: 160 } };
+        test("first monitor event has initial results reflecting existing matches in db", async () => {
+            const cachedContent = await bulkCacheContent(historyContent, true);
+            const cachedContentIds = new Set(pluckAll(cachedContent, "_id"));
 
-        let monitor$;
-        let spy;
-
-        beforeEach(async () => {
-            // create monitor with query selector hid > 160
-            monitor$ = of({ selector }).pipe(
+            // create new monitor with query selector hid > 160
+            const cutoffHid = 120;
+            const selector = { hid: { $gt: cutoffHid } };
+            const monitor$ = of({ selector }).pipe(
                 monitorQuery({ db$: content$ }),
-                take(1),
-                takeUntil(timer(2000)), // safeguard
+                takeUntil(timer(monitorSafetyTimeout)),
             );
 
-            // insert content
-            await bulkCacheContent(historyContent);
-
-            // spy on output of observable;
-            spy = new ObserverSpy();
+            // // listen to observable, wait for complete
+            const spy = new ObserverSpy();
             monitor$.subscribe(spy);
-
-            // wait for monitor to spin-up
-            await wait(monitorSpinUp);
-        });
-
-        test("first monitor event has initial results reflecting existing matches in db", async () => {
-            // wait for monitor to run out
             await spy.onComplete();
 
-            // check initial matches
-            const { initialMatches } = spy.getFirstValue();
-            expect(initialMatches).toBeDefined();
-            expect(initialMatches.length).toBeGreaterThan(0);
-            initialMatches.forEach((doc) => {
-                expect(doc.hid).toBeGreaterThan(selector.hid.$gt);
+            // should get one emit out with just the intial matches
+            const emits = spy.getValues();
+            expect(spy.getValuesLength()).toBeGreaterThan(0);
+            expect(spy.getValuesLength()).toBeLessThanOrEqual(cachedContentIds.size);
+            emits.forEach(({ doc, match, initial }) => {
+                // we aren't changing anything, results should be initial matches
+                expect(match).toBeTruthy();
+                // initial results have an identifying flag
+                expect(initial).toBe(true);
+                // every doc should match the selector
+                expect(doc => doc.hid > cutoffHid).toBeTruthy();
+                // every doc should be in the initially cached items
+                expect(cachedContentIds.has(doc._id)).toBeTruthy();
             });
         });
     });
 
     describe("updates", () => {
-        const selector = { hid: { $gt: 160 } };
 
-        let monitor$;
-        let spy;
-        let insertResults;
+        test("INSERT: adding to cache after instantiation should emit a new doc", async () => {
 
-        beforeEach(async () => {
-            monitor$ = of({ selector }).pipe(
+            const cutoffHid = 50;
+            const selector = { hid: { $gt: cutoffHid } };
+            const monitor$ = of({ selector }).pipe(
                 monitorQuery({ db$: content$ }),
-                take(2),
-                takeUntil(timer(2000)), // safeguard
+                takeUntil(timer(monitorSafetyTimeout)),
             );
 
-            // insert content
-            insertResults = await bulkCacheContent(historyContent);
-
-            // spy on output of observable;
-            spy = new ObserverSpy();
+            const spy = new ObserverSpy();
             monitor$.subscribe(spy);
-
-            // wait for monitor to spin-up
+            
+            // wait for the monitor to spin-up so the insert doesn't look like the initial vals
             await wait(monitorSpinUp);
-        });
 
-        test("new insert matching selector should emit an ADD", async () => {
-            // copy document, insert copy with some new props
-            const id = insertResults[0].id;
-            // eslint-disable-next-line no-unused-vars
-            const { _id, _rev, ...newDoc } = await getCachedContent(id);
-            newDoc.hid = 10000;
-            newDoc.id = "234234234";
-            const cacheUpdate = await cacheContent(newDoc);
-            expect(cacheUpdate.updated).toBeTruthy();
+            // Insert a row, make sure it should match the selector
+            const insertedContent = await cacheContent(historyContent[0], true);
+            expect(insertedContent.hid > cutoffHid).toBeTruthy();
 
-            // wait for monitor to run out
+            // wait for observable end
             await spy.onComplete();
 
-            // look for initial matches
-            const { initialMatches } = spy.getFirstValue();
-            expect(initialMatches).toBeDefined();
-            expect(initialMatches.length).toBeGreaterThan(0);
-
-            // make sure the updated element is NOT in the initial matches
-            const inInitialSet = initialMatches.filter((doc) => doc._id == cacheUpdate.id);
-            expect(inInitialSet.length).toEqual(0);
-
-            // check for add event
-            const { action, doc } = spy.getLastValue();
-            expect(action).toBeDefined();
-            expect(doc).toBeDefined();
-            expect(action).toEqual(ACTIONS.ADD);
-            expect(doc.hid).toEqual(newDoc.hid);
+            const emits = spy.getValues();
+            expect(spy.getValuesLength()).toEqual(1);
+            emits.forEach(({ doc, match, initial, update }) => {
+                expect(doc.hid > cutoffHid).toBeTruthy();
+                expect(match).toBe(true);
+                expect(initial).toBeUndefined();
+                expect(update).toBe(true);
+            })
         });
 
-        test("update still covered by selector should emit an UPDATE", async () => {
-            // update existing doc
-            const id = insertResults[0].id;
-            const testDoc = await getCachedContent(id);
-            testDoc.foo = 123;
-            const cacheUpdate = await cacheContent(testDoc);
-            expect(cacheUpdate.updated).toBeTruthy();
+        test("UPDATE: updating a previously emitted doc should emit an update event", async () => {
+            const cutoffHid = 50;
 
-            // wait for monitor to run out
+            // Insert a row, make sure it should match the selector
+            const insertedContent = await cacheContent(historyContent[0], true);
+            expect(insertedContent.hid > cutoffHid).toBeTruthy();
+
+            // subscribe to listener
+            const selector = { hid: { $gt: cutoffHid } };
+            const monitor$ = of({ selector }).pipe(
+                monitorQuery({ db$: content$ }),
+                takeUntil(timer(monitorSafetyTimeout)),
+            );
+            const spy = new ObserverSpy();
+            monitor$.subscribe(spy);
+            
+            // wait for the monitor to spin-up so the insert doesn't look like the initial vals
+            await wait(monitorSpinUp);
+
+            // update the content
+            const fooVal = 123;
+            insertedContent.foo = fooVal;
+            const updatedContent = await cacheContent(insertedContent, true);
+            expect(updatedContent._id).toEqual(insertedContent._id);
+            expect(updatedContent.foo).toEqual(fooVal);
+
+            // wait for observable end
             await spy.onComplete();
+            expect(spy.getValuesLength()).toEqual(2); // initial insert + later update
 
-            // look for initial matches
-            const { initialMatches } = spy.getFirstValue();
-            expect(initialMatches).toBeDefined();
-            expect(initialMatches.length).toBeGreaterThan(0);
+            // check first event
+            const firstEvent = spy.getValueAt(0);
+            {
+                const { doc, match, initial, update } = firstEvent;
+                expect(doc.foo).toBeUndefined();
+                expect(match).toBe(true);
+                expect(initial).toBe(true);
+                expect(update).toBeUndefined();
+            }
 
-            // make sure the updated element is in the initial matches
-            const inInitialSet = initialMatches.filter((doc) => doc._id == cacheUpdate.id);
-            expect(inInitialSet.length).toEqual(1);
-
-            // check for update event
-            const { action, doc } = spy.getLastValue();
-            expect(action).toBeDefined();
-            expect(doc).toBeDefined();
-            expect(action).toEqual(ACTIONS.UPDATE);
-            expect(doc.foo).toEqual(testDoc.foo);
+            // check update event
+            const secondEvent = spy.getValueAt(1);
+            {
+                const { doc, match, initial, update } = secondEvent;
+                expect(doc.foo).toBe(fooVal);
+                expect(match).toBe(true);
+                expect(initial).toBeUndefined();
+                expect(update).toBe(true);
+            }
         });
 
-        test("update no longer covered by selector should emit a REMOVE", async () => {
-            // update doc to be outside query
-            const id = insertResults[0].id;
-            const testDoc = await getCachedContent(id);
-            testDoc.hid = -3000;
-            const cacheUpdate = await cacheContent(testDoc);
-            expect(cacheUpdate.updated).toBeTruthy();
+        test("DELETE: cache update causes element to no longer be in selection", async () => {
+            const cutoffHid = 50;
 
-            // wait for monitor to run out
+            // Insert a row, make sure it should match the selector
+            const insertedContent = await cacheContent(historyContent[0], true);
+            expect(insertedContent.hid > cutoffHid).toBeTruthy();
+
+            // subscribe to listener
+            const selector = { hid: { $gt: cutoffHid } };
+            const monitor$ = of({ selector }).pipe(
+                monitorQuery({ db$: content$ }),
+                takeUntil(timer(monitorSafetyTimeout)),
+            );
+            const spy = new ObserverSpy();
+            monitor$.subscribe(spy);
+            
+            // wait for the monitor to spin-up so the insert doesn't look like the initial vals
+            await wait(monitorSpinUp);
+
+            // update the content to be outside the selector
+            const badHid = -10000;
+            insertedContent.hid = badHid; // no longer > cutoffHid
+            const updatedContent = await cacheContent(insertedContent, true);
+            expect(updatedContent._id).toEqual(insertedContent._id);
+            expect(updatedContent.hid).toEqual(badHid);
+
+            // wait for observable end
             await spy.onComplete();
+            expect(spy.getValuesLength()).toEqual(2); // initial insert + later update
 
-            // look for initial matches
-            const { initialMatches } = spy.getFirstValue();
-            expect(initialMatches).toBeDefined();
-            expect(initialMatches.length).toBeGreaterThan(0);
+            // check first event
+            const firstEvent = spy.getValueAt(0);
+            {
+                const { doc, match, initial, update } = firstEvent;
+                expect(doc).toBeDefined();
+                expect(doc).toBeInstanceOf(Object);
+                expect(doc.hid).not.toEqual(badHid);
+                expect(match).toBe(true);
+                expect(initial).toBe(true);
+                expect(update).toBeUndefined();
+            }
 
-            // make sure the updated element is in the initial matches
-            const inInitialSet = initialMatches.filter((doc) => doc._id == cacheUpdate.id);
-            expect(inInitialSet.length).toEqual(1);
-
-            // check for remove event
-            const { action, doc } = spy.getLastValue();
-            expect(action).toBeDefined();
-            expect(doc).toBeDefined();
-            expect(action).toEqual(ACTIONS.REMOVE);
-            expect(doc.hid).toEqual(testDoc.hid);
+            // check update event
+            const secondEvent = spy.getValueAt(1);
+            {
+                const { doc, match, initial, update } = secondEvent;
+                expect(doc).toBeDefined();
+                expect(doc).toBeInstanceOf(Object);
+                expect(doc.hid).toEqual(badHid);
+                expect(match).toBe(false); // no longer matches selector
+                expect(initial).toBeUndefined();
+                expect(update).toBe(true);
+            }
         });
     });
 });
 
 describe("monitorQuery: collection content", () => {
-    // pre-process raw collection content
+    // doctor and cache sample content
     const fakeParentUrl = "/abc/def/ghi";
     const testContent = collectionContent.map((doc) => {
         doc.parent_url = fakeParentUrl;
@@ -186,167 +211,188 @@ describe("monitorQuery: collection content", () => {
     });
 
     describe("initial results", () => {
-        const selector = {
-            parent_url: fakeParentUrl,
-            element_index: { $gt: 1 },
-        };
+        test("first monitor event has initial results reflecting existing matches in db", async () => {
+            // insert some stuff
+            const cachedDscContent = await bulkCacheDscContent(testContent, true);
+            const cachedContentIds = new Set(pluckAll(cachedDscContent, "_id"));
 
-        let monitor$;
-        let spy;
-
-        beforeEach(async () => {
-            monitor$ = of({ selector }).pipe(
+            // create new monitor
+            const limitIndex = 1;
+            const selector = {
+                parent_url: fakeParentUrl,
+                element_index: { $gt: limitIndex },
+            };
+            const monitor$ = of({ selector }).pipe(
                 monitorQuery({ db$: dscContent$ }),
-                take(1),
-                takeUntil(timer(2000)) // safeguard
+                takeUntil(timer(monitorSafetyTimeout))
             );
 
-            // insert content
-            await bulkCacheDscContent(testContent);
-
-            // spy on output of observable;
-            spy = new ObserverSpy();
+            // wait til done
+            const spy = new ObserverSpy();
             monitor$.subscribe(spy);
-
-            // wait for monitor to spin-up
-            await wait(monitorSpinUp);
-        });
-
-        test("first monitor event has initial results reflecting existing matches in db", async () => {
-            // wait for monitor to run out
             await spy.onComplete();
 
-            // check initial matches
-            const { initialMatches } = spy.getFirstValue();
-            expect(initialMatches).toBeDefined();
-            expect(initialMatches.length).toBeGreaterThan(0);
-            initialMatches.forEach((doc) => {
-                expect(doc.parent_url).toEqual(selector.parent_url);
-                expect(doc.element_index).toBeGreaterThan(selector.element_index.$gt);
+            // check emits
+            const emits = spy.getValues();
+            expect(spy.getValuesLength()).toBeGreaterThan(0);
+            expect(spy.getValuesLength()).toBeLessThanOrEqual(cachedContentIds.size);
+            emits.forEach(({ doc, match, initial, update }) => {
+                expect(match).toBe(true);
+                expect(initial).toBe(true);
+                expect(update).toBeUndefined();
+                expect(cachedContentIds.has(doc._id)).toBeTruthy();
+                expect(doc.element_index > limitIndex).toBeTruthy();
+                expect(doc.parent_url).toEqual(fakeParentUrl);
             });
         });
     });
 
     describe("updates", () => {
-        const selector = {
-            parent_url: fakeParentUrl,
-            name: { $regex: /M117/i },
-        };
-
-        let monitor$;
-        let insertResults;
-        let spy;
-
-        beforeEach(async () => {
-            monitor$ = of({ selector }).pipe(
+        test("INSERT: adding to cache after instantiation should emit a new doc", async () => {
+            // createa  monitor
+            const limitIndex = 0;
+            const selector = {
+                parent_url: fakeParentUrl,
+                element_index: { $gt: limitIndex },
+            };
+            const monitor$ = of({ selector }).pipe(
                 monitorQuery({ db$: dscContent$ }),
-                take(2),
-                takeUntil(timer(2000)) // safeguard
+                takeUntil(timer(monitorSafetyTimeout))
             );
-            insertResults = await bulkCacheDscContent(testContent);
-            spy = new ObserverSpy();
+
+            const spy = new ObserverSpy();
             monitor$.subscribe(spy);
 
-            // wait for monitor to spin-up
+            // wait for the monitor to spin-up so the insert doesn't look like the initial vals
             await wait(monitorSpinUp);
-        });
 
-        test("new insert matching selector should emit an ADD", async () => {
-            // insert copy of new doc
-            const newDoc = testContent[0];
-            newDoc.element_index = 10000;
-            const addResult = await cacheCollectionContent(newDoc);
-            expect(addResult.updated).toBeTruthy();
+            // Insert a row, make sure it should match the selector
+            const insertedContent = await cacheCollectionContent(testContent[3], true);
+            expect(insertedContent.element_index > limitIndex).toBeTruthy();
 
-            // check the change is in there
-            const { id: newId } = addResult;
-            const lookup = await getCachedCollectionContent(newId);
-            expect(lookup.element_index).toEqual(newDoc.element_index);
-
-            // wait for monitor to run out
+            // wait for observable end
             await spy.onComplete();
 
-            expect(spy.receivedNext()).toBe(true);
-            expect(spy.receivedComplete()).toBe(true);
-            expect(spy.getValuesLength()).toBe(2);
-
-            // look for initial matches
-            const { initialMatches } = spy.getFirstValue();
-            expect(initialMatches).toBeDefined();
-            expect(initialMatches.length).toBeGreaterThan(0);
-
-            // make sure the insert element is NOT in the initial matches
-            const inInitialSet = initialMatches.filter((doc) => doc._id == addResult.id);
-            expect(inInitialSet.length).toEqual(0);
-
-            // look for add event
-            const { action, doc } = spy.getLastValue();
-            expect(action).toBeDefined();
-            expect(doc).toBeDefined();
-            expect(action).toEqual(ACTIONS.ADD);
-            expect(doc.element_index).toEqual(newDoc.element_index);
+            const emits = spy.getValues();
+            expect(spy.getValuesLength()).toEqual(1);
+            emits.forEach(({ doc, match, initial, update }) => {
+                expect(doc.element_index > limitIndex).toBeTruthy();
+                expect(doc._id).toEqual(insertedContent._id);
+                expect(match).toBe(true);
+                expect(initial).toBeUndefined();
+                expect(update).toBe(true);
+            });
         });
 
-        test("update still covered by selector should emit an UPDATE", async () => {
-            // look up an existing doc
-            const id = insertResults[0].id;
-            const lookupDoc = await getCachedCollectionContent(id);
-            expect(lookupDoc).toBeDefined();
-            expect(lookupDoc._id).toEqual(id);
+        test("UPDATE: updating a previously emitted doc should emit an update event", async () => {
+            const limitIndex = 0;
 
-            // update it
-            lookupDoc.foo = "fakestuff";
-            const updateResult = await cacheCollectionContent(lookupDoc);
-            expect(updateResult.updated).toBeTruthy();
+            // Insert a row, make sure it should match the selector
+            const insertedContent = await cacheCollectionContent(testContent[1], true);
+            expect(insertedContent.element_index > limitIndex).toBeTruthy();
 
-            // wait for monitor to run out
+            // create  monitor
+            const selector = {
+                parent_url: fakeParentUrl,
+                element_index: { $gt: limitIndex },
+            };
+            const monitor$ = of({ selector }).pipe(
+                monitorQuery({ db$: dscContent$ }),
+                takeUntil(timer(monitorSafetyTimeout))
+            );
+
+            const spy = new ObserverSpy();
+            monitor$.subscribe(spy);
+
+            // wait for the monitor to spin-up so the insert doesn't look like the initial vals
+            await wait(monitorSpinUp);
+
+            // update the content
+            const fooVal = 123;
+            insertedContent.foo = fooVal;
+            const updatedContent = await cacheCollectionContent(insertedContent, true);
+            expect(updatedContent._id).toEqual(insertedContent._id);
+            expect(updatedContent.foo).toEqual(fooVal);
+
+            // wait for observable end
             await spy.onComplete();
+            expect(spy.getValuesLength()).toEqual(2); // initial insert + later update
 
-            // look for initial matches
-            const { initialMatches } = spy.getFirstValue();
-            expect(initialMatches).toBeDefined();
-            expect(initialMatches.length).toBeGreaterThan(0);
+            // check first event
+            const firstEvent = spy.getValueAt(0);
+            {
+                const { doc, match, initial, update } = firstEvent;
+                expect(doc.foo).toBeUndefined();
+                expect(match).toBe(true);
+                expect(initial).toBe(true);
+                expect(update).toBeUndefined();
+            }
 
-            // make sure the updated doc is in the initial matches
-            const inInitialSet = initialMatches.filter((doc) => doc._id == updateResult.id);
-            expect(inInitialSet.length).toEqual(1);
-
-            // look for add event
-            const { action, doc } = spy.getLastValue();
-            expect(action).toBeDefined();
-            expect(doc).toBeDefined();
-            expect(action).toEqual(ACTIONS.UPDATE);
-            expect(doc.foo).toEqual(lookupDoc.foo);
+            // check update event
+            const secondEvent = spy.getValueAt(1);
+            {
+                const { doc, match, initial, update } = secondEvent;
+                expect(doc.foo).toBe(fooVal);
+                expect(match).toBe(true);
+                expect(initial).toBeUndefined();
+                expect(update).toBe(true);
+            }
         });
 
-        test("update no longer covered by selector should emit a REMOVE", async () => {
-            // set to deleted
-            const id = insertResults[0].id;
-            const testDoc = await getCachedCollectionContent(id);
-            testDoc.name = "ABC";
-            testDoc.element_identifier = "ABC";
-            const cacheUpdate = await cacheCollectionContent(testDoc);
-            expect(cacheUpdate.updated).toBeTruthy();
+        test("DELETE: cache update causes element to no longer be in selection", async () => {
+            const limitIndex = 0;
 
-            // wait for monitor to run out
+            // Insert a row, make sure it should match the selector
+            const insertedContent = await cacheCollectionContent(testContent[1], true);
+            expect(insertedContent.element_index > limitIndex).toBeTruthy();
+
+            // create  monitor
+            const selector = {
+                parent_url: fakeParentUrl,
+                element_index: { $gt: limitIndex },
+            };
+            const monitor$ = of({ selector }).pipe(
+                monitorQuery({ db$: dscContent$ }),
+                takeUntil(timer(monitorSafetyTimeout))
+            );
+
+            const spy = new ObserverSpy();
+            monitor$.subscribe(spy);
+
+            // update the content to be outside the selector
+            const badIndex = -10000;
+            insertedContent.element_index = badIndex; // no longer > cutoffHid
+            const updatedContent = await cacheCollectionContent(insertedContent, true);
+            expect(updatedContent._id).toEqual(insertedContent._id);
+            expect(updatedContent.element_index).toEqual(badIndex);
+
+            // wait for observable end
             await spy.onComplete();
+            expect(spy.getValuesLength()).toEqual(2); // initial insert + later update
 
-            // look for initial matches
-            const { initialMatches } = spy.getFirstValue();
-            expect(initialMatches).toBeDefined();
-            expect(initialMatches.length).toBeGreaterThan(0);
+            // check first event
+            const firstEvent = spy.getValueAt(0);
+            {
+                const { doc, match, initial, update } = firstEvent;
+                expect(doc).toBeDefined();
+                expect(doc).toBeInstanceOf(Object);
+                expect(doc.element_index).not.toEqual(badIndex);
+                expect(match).toBe(true);
+                expect(initial).toBe(true);
+                expect(update).toBeUndefined();
+            }
 
-            // make sure the updated element is in the initial matches
-            const inInitialSet = initialMatches.filter((doc) => doc._id == cacheUpdate.id);
-            expect(inInitialSet.length).toEqual(1);
-
-            // look for the remove event
-            const { action, doc } = spy.getLastValue();
-            expect(action).toBeDefined();
-            expect(doc).toBeDefined();
-            expect(action).toEqual(ACTIONS.REMOVE);
-            expect(doc.name).toEqual(testDoc.name);
-            expect(doc.element_identifier).toEqual(testDoc.element_identifier);
+            // check update event
+            const secondEvent = spy.getValueAt(1);
+            {
+                const { doc, match, initial, update } = secondEvent;
+                expect(doc).toBeDefined();
+                expect(doc).toBeInstanceOf(Object);
+                expect(doc.element_index).toEqual(badIndex);
+                expect(match).toBe(false); // no longer matches selector
+                expect(initial).toBeUndefined();
+                expect(update).toBe(true);
+            }
         });
     });
 });

client/src/components/History/caching/db/pouch.js

@@ -7,7 +7,7 @@ import moment from "moment";
 import deepEqual from "deep-equal";
 import { defer, pipe, from } from "rxjs";
 import { tap, filter, mergeMap, reduce, shareReplay } from "rxjs/operators";
-import { needs } from "../operators/needs";
+import { needs } from "utils/observable";
 import { dasherize } from "underscore.string";
 
 import PouchDB from "pouchdb";

client/src/components/History/caching/enums.js

@@ -0,0 +1,9 @@
+// seek types
+
+const seeks = { ASC: "asc", DESC: "desc" };
+
+seeks.isValid = (val) => {
+    return Object.values(SEEK).some((v) => v == val);
+};
+
+export const SEEK = seeks;

client/src/components/History/caching/index.js

@@ -7,12 +7,14 @@ import { toPromise, toOperator } from "./workerClient";
 export const monitorContentQuery = toOperator("monitorContentQuery");
 export const monitorDscQuery = toOperator("monitorDscQuery");
 export const monitorHistoryContent = toOperator("monitorHistoryContent");
+export const monitorCollectionContent = toOperator("monitorCollectionContent");
 
 /**
  * Loaders
  */
 export const loadHistoryContents = toOperator("loadHistoryContents");
 export const loadDscContent = toOperator("loadDscContent");
+export const collectionContentAjaxLoad = toOperator("collectionContentAjaxLoad");
 
 /**
  * Cache promise functions

client/src/components/History/caching/loadDscContent.js

@@ -3,70 +3,71 @@
  * or filters data in the listing for immediate lookups. All of them will run
  * one or more ajax calls against the api and cache ther results.
  */
-import { map, withLatestFrom, pluck } from "rxjs/operators";
-import { throttleDistinct } from "utils/observable/throttleDistinct";
-import { nth } from "utils/observable/nth";
+import { map, pluck, publish } from "rxjs/operators";
+import { hydrate, nth, whenAny } from "utils/observable";
+import { throttleDistinct } from "./operators/throttleDistinct";
+import { requestWithUpdateTime } from "./operators/requestWithUpdateTime";
 
 import { bulkCacheDscContent } from "./db";
-import { prependPath } from "./workerConfig";
-import { hydrate } from "./operators/hydrate";
-import { chunkParam } from "./operators/chunk";
-import { requestWithUpdateTime } from "./operators/requestWithUpdateTime";
-import { summarizeCacheOperation, dateStore } from "./loadHistoryContents";
 import { SearchParams } from "../model/SearchParams";
+import { ScrollPos } from "../model/ScrollPos";
+import { prependPath } from "./workerConfig";
+import { summarizeCacheOperation, dateStore } from "./loadHistoryContents";
 
 /**
  * Load collection content (drill down)
  * Params: contents_url + search params + element_index
  */
 // prettier-ignore
-export const loadDscContent = (cfg = {}) => (src$) => {
+export const loadDscContent = (cfg = {}) => {
 
-    const {
-        onceEvery = 10 * 1000,
-        chunkSize = SearchParams.pageSize
-    } = cfg;
+    return publish(src$ => {
+        // clean and chunk inputs
+        const inputs$ = src$.pipe(
+            hydrate([undefined, SearchParams, ScrollPos]),
+        );
 
-    // clean and chunk inputs
-    const inputs$ = src$.pipe(
-        hydrate([undefined, SearchParams]),
-        chunkParam(2, chunkSize)
-    );
+        // add contents_url to cache data, will use it as a key
+        const url$ = inputs$.pipe(nth(0));
 
-    // add contents_url to cache data, will use it as a key
-    const url$ = inputs$.pipe(nth(0));
+        // request, throttle frequently repeated requests
+        const ajaxResponse$ = inputs$.pipe(
+            collectionContentAjaxLoad(cfg)
+        );
 
-    // request, throttle frequently repeated requests
-    const ajaxResponse$ = inputs$.pipe(
+        // Add in the url we found this content at because that is going to be like
+        // a parent_id in the database
+        const response$ = whenAny(ajaxResponse$, url$).pipe(
+            map(([response, url]) => response.map((row) => ({ ...row, parent_url: url })))
+        );
+
+        const cached$ = response$.pipe(
+            bulkCacheDscContent(), 
+            summarizeCacheOperation(),
+        );
+
+        return cached$;
+    });
+};
+
+// Separating this out so it's easier to mock for testing purposes
+export const collectionContentAjaxLoad = (cfg = {}) => (inputs$) => {
+    const { onceEvery = 10 * 1000 } = cfg;
+    return inputs$.pipe(
         map(buildDscContentUrl),
         throttleDistinct({ timeout: onceEvery }),
         map(prependPath),
         requestWithUpdateTime({ dateStore }),
         pluck("response")
     );
-
-    // Add in the url we found this content at because that is going to be like
-    // a parent_id in the database
-    const response$ = ajaxResponse$.pipe(
-        withLatestFrom(url$),
-        map(([response, url]) => response.map((row) => ({ ...row, parent_url: url })))
-    );
-
-    const cached$ = response$.pipe(
-        bulkCacheDscContent(),
-        summarizeCacheOperation()
-    );
-
-    return cached$; // enqueue(load$, "load contents");
 };
 
 // Collection + params -> request url w/o update_time
 // ignore params for now, we don't filter collection contents yet.
-export const buildDscContentUrl = (inputs) => {
+const buildDscContentUrl = (inputs) => {
     // eslint-disable-next-line no-unused-vars
-    const [url, filters, cursor] = inputs;
-    const limit = SearchParams.pageSize;
-    const skipClause = cursor ? `offset=${cursor}` : "";
+    const [url, filters, { offset = 0, limit = SearchParams.pageSize }] = inputs;
+    const skipClause = offset ? `offset=${offset}` : "";
     const limitClause = limit ? `limit=${limit}` : "";
     const qs = [skipClause, limitClause].filter((o) => o.length).join("&");
     return `${url}?${qs}`;

client/src/components/History/caching/loadHistoryContents.js

@@ -1,10 +1,7 @@
-import { pipe } from "rxjs";
-import { map, withLatestFrom, pluck, shareReplay } from "rxjs/operators";
-import { throttleDistinct } from "utils/observable/throttleDistinct";
-import { tag } from "rxjs-spy/operators/tag";
-
-import { hydrate } from "./operators/hydrate";
-import { requestWithUpdateTime, requestWithUpdateTimeNoInitial } from "./operators/requestWithUpdateTime";
+import { zip } from "rxjs";
+import { map, pluck, share } from "rxjs/operators";
+import { hydrate } from "utils/observable";
+import { requestWithUpdateTime } from "./operators/requestWithUpdateTime";
 import { prependPath } from "./workerConfig";
 import { bulkCacheContent } from "./db";
 import { SearchParams } from "../model/SearchParams";
@@ -19,79 +16,63 @@ export const clearHistoryDateStore = async () => {
 };
 
 /**
- * Turn [historyId, params, hid] into content requests. Cache results and send
+ * Turn [historyId, params, pagination] into content requests. Cache results and send
  * back statistics about what was returned for this query.
  */
 // prettier-ignore
 export const loadHistoryContents = (cfg = {}) => (rawInputs$) => {
-    const {
-        onceEvery = 30 * 1000,
-        windowSize = SearchParams.pageSize,
-        noInitial = false
+    const { 
+        noInitial = false,
+        windowSize = SearchParams.pageSize
     } = cfg;
 
-    // if noInitial, only returns updates
-    const dateAppender = noInitial ? requestWithUpdateTimeNoInitial : requestWithUpdateTime
-
     const inputs$ = rawInputs$.pipe(
-        shareReplay(1)
+        hydrate([undefined, SearchParams]),
     );
 
     const ajaxResponse$ = inputs$.pipe(
-        hydrate([undefined, SearchParams]),
         map(([id, params, hid]) => {
             const baseUrl = `/api/histories/${id}/contents/near/${hid}/${windowSize}`;
             return `${baseUrl}?${params.historyContentQueryString}`;
         }),
-        throttleDistinct({ timeout: onceEvery }),
         map(prependPath),
-        dateAppender({ dateStore }),
-        shareReplay(1),
+        requestWithUpdateTime({ dateStore, noInitial }),
+        share(),
     );
 
     const cacheSummary$ = ajaxResponse$.pipe(
         pluck("response"),
         bulkCacheContent(),
         summarizeCacheOperation(),
-        tag("loadHistoryContents-cacheSummary"),
     );
 
-    return cacheSummary$.pipe(
-        withLatestFrom(ajaxResponse$, inputs$),
-        map(([summary, ajaxResponse, inputs]) => {
-
-            // actual list
+    return zip(ajaxResponse$, cacheSummary$).pipe(
+        map(([ajaxResponse, summary]) => {
             const { xhr, response = [] } = ajaxResponse;
             const { max: maxContentHid, min: minContentHid } = getPropRange(response, "hid");
+            const headerInt = field => parseInt(xhr.getResponseHeader(field));
 
             // header counts
-            const matchesUp = +xhr.getResponseHeader("matches_up");
-            const matchesDown = +xhr.getResponseHeader("matches_down");
-            const totalMatchesUp = +xhr.getResponseHeader("total_matches_up");
-            const totalMatchesDown = +xhr.getResponseHeader("total_matches_down");
-            const minHid = +xhr.getResponseHeader("min_hid");
-            const maxHid = +xhr.getResponseHeader("max_hid");
+            const matchesUp = headerInt("matches_up");
+            const matchesDown = headerInt("matches_down");
+            const totalMatchesUp = headerInt("total_matches_up");
+            const totalMatchesDown = headerInt("total_matches_down");
+            const minHid = headerInt("min_hid");
+            const maxHid = headerInt("max_hid");
 
             return {
-                // original inputs
-                inputs,
-
-                // cache summary
                 summary,
-
-                // derived from the returned content list
-                minContentHid,
-                maxContentHid,
-
-                // ajax result stats
+                matches: matchesUp + matchesDown,
+                totalMatches: totalMatchesUp + totalMatchesDown,
                 minHid,
                 maxHid,
+                minContentHid, // minimum hid in the returned result
+                maxContentHid, // maximum hid in the returned result
+
                 matchesUp,
                 matchesDown,
-                matches: matchesUp + matchesDown,
                 totalMatchesUp,
                 totalMatchesDown,
-                totalMatches: totalMatchesUp + totalMatchesDown
             };
         })
     );
@@ -103,15 +84,15 @@ export const loadHistoryContents = (cfg = {}) => (rawInputs$) => {
  * summarizes what pouchdb did during the bulk cache and sends back some stats.
  */
 // prettier-ignore
-export const summarizeCacheOperation = () => pipe(
-    map((list) => {
+export const summarizeCacheOperation = () => {
+    return map((list) => {
         const cached = list.filter((result) => result.updated || result.ok);
         return {
             updatedItems: cached.length,
             totalReceived: list.length,
         };
     })
-);
+}
 
 /**
  * Gets min and max values from an array of objects

client/src/components/History/caching/monitorHistoryContent.js

@@ -1,10 +1,33 @@
-import { merge, partition, concat } from "rxjs";
-import { map, share, shareReplay, pluck, take, reduce, distinctUntilChanged } from "rxjs/operators";
-import { content$, buildContentId } from "./db/observables";
-import { monitorQuery, ACTIONS } from "./db/monitorQuery";
-import { hydrate } from "./operators/hydrate";
+import { merge } from "rxjs";
+import { map, distinctUntilChanged } from "rxjs/operators";
+import { content$, dscContent$, buildContentId, buildCollectionId } from "./db/observables";
+import { monitorQuery } from "./db/monitorQuery";
 import { SearchParams } from "../model/SearchParams";
 import { deepEqual } from "deep-equal";
+import { hydrate, shareButDie, show } from "utils/observable";
+import { SEEK } from "./enums";
+
+// history contents monitor
+export const monitorHistoryContent = (cfg = {}) => {
+    const { aggregationKeyField = "hid", ...moreCfg } = cfg;
+    return twoWayMonitor({
+        ...moreCfg,
+        db$: content$,
+        buildRequest: buildContentPouchRequest,
+        aggregationKeyField,
+    });
+};
+
+// collection contents monitor
+export const monitorCollectionContent = (cfg = {}) => {
+    const { aggregationKeyField = "element_index", ...moreCfg } = cfg;
+    return twoWayMonitor({
+        ...moreCfg,
+        db$: dscContent$,
+        buildRequest: buildCollectionPouchRequest,
+        aggregationKeyField,
+    });
+};
 
 /**
  * Search cache upward and downward from the scroll HID, filtering out results
@@ -13,68 +36,58 @@ import { deepEqual } from "deep-equal";
  * contents if the user is rapidly dragging the scrollbar to different regions
  */
 // prettier-ignore
-export const monitorHistoryContent = (cfg = {}) => (src$) => {
-    // console.warn("monitorHistoryContent operator", cfg);
-
+export const twoWayMonitor = (cfg = {}) => (src$) => {
     const {
-        db$ = content$,
-        pages = 1,
+        db$,
+        aggregationKeyField,
+        buildRequest = buildContentPouchRequest,
         pageSize = SearchParams.pageSize,
-        bench = pages * pageSize,
-        limit = pages * pageSize,
-        inputDebounce = 0,
+        inputDebounce = 100,
+        debug = false,
     } = cfg;
 
     const input$ = src$.pipe(
-        hydrate([undefined, SearchParams]),
-        shareReplay(1),
+        show(debug, ([id, filters, hid]) => console.log("twoWayMonitor inputs", id, hid)),
+        hydrate([undefined, SearchParams]), 
+        shareButDie(1)
     );
 
+    // one page up
     const up$ = input$.pipe(
-        map(buildContentPouchRequest({
-            seek: "asc",
-            limit: bench
-        })),
+        map(buildRequest({ seek: SEEK.ASC, pageSize })),
         distinctUntilChanged(deepEqual),
-        monitorQuery({
-            db$,
-            inputDebounce,
-        }),
+        show(debug, (req) => console.log("twoWayMonitor request (up)", req.selector)),
+        monitorQuery({ db$, inputDebounce, debug, label: "up" }),
+        show(debug, ({ doc, match }) => console.log("twoWayMonitor response (up)", doc[aggregationKeyField], match)),
     );
 
+    // current page + next page = 2 pages down
     const down$ = input$.pipe(
-        map(buildContentPouchRequest({
-            seek: "desc",
-            limit
-        })),
+        map(buildRequest({ seek: SEEK.DESC, pageSize })),
         distinctUntilChanged(deepEqual),
-        monitorQuery({
-            db$,
-            inputDebounce,
+        show(debug, (req) => console.log("twoWayMonitor request (down)", req.selector)),
+        monitorQuery({ db$, inputDebounce, debug, label: "down" }),
+        show(debug, ({ doc, match }) => console.log("twoWayMonitor response (down)", doc[aggregationKeyField], match)),
+    );
+
+    const change$ = merge(up$, down$).pipe(
+        map((response) => {
+            // don't bother sending back non-matching results
+            // we're just going to delete them from the skiplist
+            // using the aggregation key, so no need to serialize them
+            const { match, doc, ...more} = response;
+            const key = doc[aggregationKeyField];
+            return match ? { key, match, doc, ...more } : { key, match, ...more };
+        }),
+        show(debug, update => {
+            const { key, match } = update;
+            console.log("monitorHistoryContent change", key, match);
         }),
     );
 
-
-    // merge both up and down initial events into a single initial event
-    // (reduces bouncing in the output)
-
-    const all$ = merge(up$, down$).pipe(share());
-    const splitter = evt => evt.action == ACTIONS.INITIAL;
-    const [ allInital$, update$ ] = partition(all$, splitter);
-
-    const initial$ = allInital$.pipe(
-        pluck('initialMatches'),
-        take(2),
-        reduce((allDocs, docs) => allDocs.concat(docs), []),
-        map(docs => ({ action: ACTIONS.INITIAL, initialMatches: docs })),
-    );
-
-    return concat(initial$, update$);
+    return change$;
 };
 
-// seek types
-export const SEEK = { ASC: "asc", DESC: "desc" };
-
 /**
  * Generate a PouchDB selector for a specified history, filters, and a rough
  * guess of where to start looking.
@@ -83,19 +96,25 @@ export const SEEK = { ASC: "asc", DESC: "desc" };
  * and we need to be able to select without loading everything
  */
 export const buildContentPouchRequest = (cfg = {}) => (inputs) => {
-    const { limit = SearchParams.pageSize, seek = SEEK.DESC } = cfg;
+    const { seek, pageSize = SearchParams.pageSize } = cfg;
     const [history_id, params, hid] = inputs;
 
     // look up or down from target hid
     const targetId = buildContentId({ history_id, hid });
+
+    // SEEK.ASC means the top seek, HID > target
     const comparator = seek == SEEK.ASC ? "$gt" : "$lte";
 
+    // one page above the target, then 2 after to get the current page and some buffer
+    const pages = seek == SEEK.ASC ? 1 : 2;
+    const limit = pages * pageSize;
+
     // index, will build if not existent
     const filterFields = Array.from(params.criteria.keys()).map((f) => params.getPouchFieldName(f));
     const fields = ["_id", "history_id", ...filterFields];
     const ddoc = "idx-" + fields.sort().join("-");
 
-    return {
+    const request = {
         selector: {
             $and: [
                 // doc id + direction (up/down)
@@ -111,4 +130,43 @@ export const buildContentPouchRequest = (cfg = {}) => (inputs) => {
             ddoc,
         },
     };
+
+    return request;
+};
+
+// filters currently unused, but they could be if we add some filter UI
+export const buildCollectionPouchRequest = (cfg = {}) => (inputs) => {
+    const { seek, pageSize = SearchParams.pageSize } = cfg;
+
+    // eslint-disable-next-line no-unused-vars
+    const [parent_url, filters, element_index] = inputs;
+    const targetId = buildCollectionId({ parent_url, element_index });
+
+    // SEEK.ASC means above the target means element_index < target
+    const comparator = seek == SEEK.ASC ? "$lt" : "$gte";
+
+    // 3 pages total, 1 before, the current one, 1 after;
+    const pages = seek == SEEK.ASC ? 1 : 2;
+    const limit = pages * pageSize;
+
+    // when searching upward we invert the sort because we want the
+    // results closest to the target
+    const idSort = seek == SEEK.ASC ? "desc" : "asc";
+
+    const request = {
+        selector: {
+            _id: { [comparator]: targetId },
+            parent_url: { $eq: parent_url },
+        },
+        sort: [{ _id: idSort }],
+        limit,
+        index: {
+            fields: ["_id"],
+            sort: [{ _id: "asc" }],
+            name: "collection contents by parent_url and element_index ascending",
+            ddoc: "idx-dsc-contents-by-url-and-element-asc",
+        },
+    };
+
+    return request;
 };

client/src/components/History/caching/monitorHistoryContent.test.js

@@ -1,24 +1,28 @@
 import { timer, of } from "rxjs";
-import { take, pluck, takeUntil } from "rxjs/operators";
+import { takeUntil } from "rxjs/operators";
 import { firstValueFrom } from "utils/observable/firstValueFrom";
 import { wipeDatabase } from "./db/wipeDatabase";
 import { wait } from "jest/helpers";
 import { ObserverSpy } from "@hirez_io/observer-spy";
 
 import { SearchParams } from "../model/SearchParams";
-import { content$, buildContentId } from "../caching/db/observables";
-import { bulkCacheContent, getCachedContent, cacheContent } from "../caching/db/promises";
+import { content$ } from "../caching/db/observables";
+import { bulkCacheContent, cacheContent } from "../caching/db/promises";
 import { find } from "../caching/db/find";
 import { buildContentPouchRequest, monitorHistoryContent } from "./monitorHistoryContent";
-import { ACTIONS } from "../caching/db/monitorQuery";
 
 // test data
 import historyContent from "../test/json/historyContent.json";
 // import collectionContent from "../../test/json/collectionContent.json";
 
+jest.mock("app");
+jest.mock("../caching");
+
 beforeEach(wipeDatabase);
 afterEach(wipeDatabase);
 
+const monitorSpinUp = 100;
+const monitorSafetyTimeout = 1000;
 const selectorHasField = (selector, field) => selector.$and.some((row) => row[field] !== undefined);
 
 describe("buildContentPouchRequest", () => {
@@ -39,7 +43,16 @@ describe("buildContentPouchRequest", () => {
 });
 
 describe("monitorHistoryContent", () => {
-    beforeEach(async () => await bulkCacheContent(historyContent));
+    let cachedContentMap;
+
+    // cache sample content, store in a map for later, keyed by HID
+    beforeEach(async () => {
+        const cachedContent = await bulkCacheContent(historyContent, true);
+        const cachedContentEntries = cachedContent.map((o) => [o.hid, o]);
+        cachedContentMap = new Map(cachedContentEntries);
+    });
+
+    afterEach(() => (cachedContentMap = null));
 
     test("sanity check all the data is in there", async () => {
         const { history_id } = historyContent[0];
@@ -54,28 +67,25 @@ describe("monitorHistoryContent", () => {
         const monitor$ = of([history_id, new SearchParams(), hid]).pipe(
             monitorHistoryContent({
                 inputDebounce: 50,
-                pageSize: 100000000, // get everything without pagination for now
+                pageSize: 10, // get everything without pagination for now
             }),
-            pluck("initialMatches"),
-            take(1),
-            takeUntil(timer(2000)) // safety
+            takeUntil(timer(monitorSafetyTimeout)) // safety
         );
 
         const spy = new ObserverSpy();
         monitor$.subscribe(spy);
-
         await spy.onComplete();
+        expect(spy.getValuesLength()).toBeGreaterThan(0);
 
-        // monitorHistoryContent has 2 seeks, one up, one down, so two emits
-        // with initialMatches, some of the test rows have been doctored so they
-        // don't match the initial filters
-        const evts = spy.getValues();
-        const matches = evts.reduce((acc, matches) => acc + matches.length, 0);
-
-        // show only visible, undeleted rows, there are a couple that don't match
-        const expectedRows = historyContent.filter((row) => row.deleted == false && row.visible == true);
-        expect(expectedRows.length).not.toEqual(historyContent.length);
-        expect(matches).toEqual(expectedRows.length);
+        spy.getValues().forEach((evt) => {
+            const { key, match, doc } = evt;
+            expect(match).toBe(true);
+            expect(doc.hid).toEqual(key);
+            expect(cachedContentMap.has(key)).toBe(true);
+            // we had default params
+            expect(doc.isDeleted).toBe(false);
+            expect(doc.visible).toBe(true);
+        });
     });
 
     test("should see subsequent updates", async () => {
@@ -85,37 +95,29 @@ describe("monitorHistoryContent", () => {
 
         const monitor$ = of(inputs).pipe(
             monitorHistoryContent({ inputDebounce: 50 }),
-            take(2),
-            takeUntil(timer(2000)) // safety, ends stream if test fails
+            takeUntil(timer(monitorSafetyTimeout)) // safety
         );
 
         // monitor observable
         const spy = new ObserverSpy();
         monitor$.subscribe(spy);
 
-        // wait for first emission before updating
-        await wait(500);
+        // wait for first emission before updating, this lets all
+        // the initial matches emit
+        await wait(monitorSpinUp);
 
-        // update doc
-        const lookup = await getCachedContent(buildContentId(firstDoc));
-        lookup.foobar = 123;
-        const cached = await cacheContent(lookup);
-        expect(cached.updated).toBeTruthy();
+        // update one doc
+        const firstHid = historyContent[0].hid;
+        const updateMe = cachedContentMap.get(firstHid);
+        const fakeVal = 123;
+        updateMe.foobar = fakeVal;
+        const updatedContent = await cacheContent(updateMe, true);
+        expect(updatedContent.foobar).toEqual(fakeVal);
 
         // wait until monitor completes
         await spy.onComplete();
-        expect(spy.getValuesLength()).toEqual(2);
-        expect(spy.receivedError()).toBe(false);
-        expect(spy.receivedNext()).toBe(true);
-        expect(spy.receivedComplete()).toBe(true);
 
-        // first emission should have been the initial load
-        const firstUpdate = spy.getFirstValue();
-        expect(firstUpdate.action).toEqual(ACTIONS.INITIAL);
-
-        const lastUpdate = spy.getLastValue();
-        expect(lastUpdate.action).toEqual(ACTIONS.UPDATE);
-        expect(lastUpdate.doc._id).toEqual(lookup._id);
-        expect(lastUpdate.doc.foobar).toEqual(123);
+        const lastEmit = spy.getLastValue();
+        expect(lastEmit.doc.foobar).toEqual(fakeVal);
     });
 });

client/src/components/History/caching/operators/requestWithUpdateTime.js

@@ -23,10 +23,17 @@ export const requestWithUpdateTime = (config = {}) => {
         bufferSeconds = 0,
         dateFieldName = "update_time",
         requestTime = moment.utc(),
+        // indicates we don't want initial results
+        noInitial = false
     } = config;
 
     // mark and flag this update-time, append to next request with same base
     return pipe(
+        tap((url) => {
+            if (noInitial && !dateStore.has(url)) {
+                dateStore.set(url, moment.utc());
+            }
+        }),
         mergeMap((baseUrl) => of(baseUrl).pipe(
             appendUpdateTime({ dateStore, bufferSeconds, dateFieldName }),
             mergeMap(ajax),
@@ -35,22 +42,6 @@ export const requestWithUpdateTime = (config = {}) => {
     );
 };
 
-// prettier-ignore
-// Like request withUpdateTime but omits initial results
-export const requestWithUpdateTimeNoInitial = (config = {}) => {
-    const {
-        dateStore = requestDateStore,
-    } = config;
-    return pipe(
-        tap((url) => {
-            if (!dateStore.has(url)) {
-                dateStore.set(url,  moment.utc());
-            }
-        }),
-        requestWithUpdateTime(config)
-    );
-}
-
 /**
  * Takes a base URL appends an update_time-gt restriction based on the lst
  * time this URL was requestd as indicated by the dateStore.
@@ -60,7 +51,7 @@ export const requestWithUpdateTimeNoInitial = (config = {}) => {
 const appendUpdateTime = (cfg = {}) => {
     const {
         dateStore = requestDateStore,
-        dateFieldName = "update_time"
+        dateFieldName = "update_time",
     } = cfg;
 
     return pipe(

client/src/components/History/caching/operators/throttleDistinct.js

@@ -4,7 +4,7 @@
 import moment from "moment";
 import { pipe } from "rxjs";
 import { filter } from "rxjs/operators";
-import { createDateStore } from "../model/DateStore";
+import { createDateStore } from "../../model/DateStore";
 
 export const throttleDistinctDateStore = createDateStore("throttleDistinct default");
 

client/src/components/History/model/ScrollPos.js

@@ -0,0 +1,16 @@
+export class ScrollPos {
+    constructor(props = {}) {
+        const { cursor = 0.0, key = null } = props;
+        this.cursor = cursor;
+        this.key = key;
+    }
+}
+
+ScrollPos.create = function (props) {
+    const o = new ScrollPos(props);
+    return Object.freeze(o);
+};
+
+ScrollPos.equals = function (a, b) {
+    return a.cursor == b.cursor && a.key == b.key;
+};

client/src/components/History/model/SearchParams.js

@@ -1,4 +1,3 @@
-import config from "config";
 import deepEqual from "deep-equal";
 import { isString } from "underscore";
 
@@ -126,43 +125,68 @@ export class SearchParams {
         return filters;
     }
 
-    // render a query string for use in querying content from the contnts/near endoint
+    // Creates an object of query criteria for use with the api
+    get historyContentQueryFields() {
+        return Array.from(this.criteria)
+            .map(([userField, val]) => {
+                let serverField = this.getServerFieldName(userField);
+                let serverVal = val;
+
+                switch (serverField) {
+                    // some client-side filters do not correspond to filters on the server
+                    // they can be used to filter local results but will not affect the polling
+                    // TODO: consider adding them as available filter options on the api?
+                    case null:
+                        return [];
+
+                    // This is advertised to work, but is broken on the current api
+                    case "annotation":
+                    case "description":
+                        return [];
+
+                    // non-standard REST bools
+                    // deleted serverField was reserved by pouchDB, needed to rename it to "isDeleted"
+                    case "deleted":
+                    case "visible":
+                        serverVal = dumbBool(val);
+                        break;
+
+                    // no text searching in some fields
+                    case "hid":
+                    case "state":
+                    case "history_content_type":
+                    case "type":
+                        break;
+
+                    // assume text-contains search
+                    default:
+                        serverField = serverField + "-contains";
+                        serverVal = encodeURIComponent(val);
+                        break;
+                }
+
+                return [serverField, serverVal];
+            })
+            .filter((pair) => pair.length == 2)
+            .reduce((fields, [k, v]) => {
+                fields[k] = v;
+                return fields;
+            }, {});
+    }
+
+    // legacy q/qv syntax for content api
+    // TODO: Delete when we no longer use q/qv
+    get legacyContentQueryString() {
+        return Object.entries(this.historyContentQueryFields)
+            .map(([f, v]) => `q=${f}&qv=${v}`)
+            .join("&");
+    }
+
+    // Standard query string (field=val&field=val...)
     get historyContentQueryString() {
-        const parts = Array.from(this.criteria).map(([userField, val]) => {
-            const serverField = this.getServerFieldName(userField);
-
-            switch (serverField) {
-                // some client-side filters do not correspond to filters on the server
-                // they can be used to filter local results but will not affect the polling
-                // TODO: consider adding them as available filter options on the api?
-                case null:
-                    return "";
-
-                // This is advertised to work, but is broken on the current api
-                case "annotation":
-                case "description":
-                    return "";
-
-                // non-standard REST bools
-                // deleted serverField was reserved by pouchDB, needed to rename it to "isDeleted"
-                case "deleted":
-                case "visible":
-                    return `${serverField}=${dumbBool(val)}`;
-
-                // no text searching in some fields
-                case "hid":
-                case "state":
-                case "history_content_type":
-                case "type":
-                    return `${serverField}=${val}`;
-
-                // assume text-contains search
-                default:
-                    return `${serverField}-contains=${encodeURIComponent(val)}`;
-            }
-        });
-
-        return parts.filter((o) => o.length).join("&");
+        return Object.entries(this.historyContentQueryFields)
+            .map(([f, v]) => `${f}=${v}`)
+            .join("&");
     }
 
     // maps friendly user field name to internal data field if necessary
@@ -177,12 +201,10 @@ export class SearchParams {
     }
 
     // output current state to log
-    report(label = "params") {
+    report(label = "params", collapsed = true) {
         const { showDeleted, showHidden, filterText } = this;
-        const dString = showDeleted ? "showDeleted" : "";
-        const hString = showHidden ? "showHidden" : "";
-
-        console.groupCollapsed(label, `(${dString} ${hString}`);
+        const consoleOpen = collapsed ? console.groupCollapsed : console.group;
+        consoleOpen.call(console, label);
         console.log("showDeleted", showDeleted);
         console.log("showHidden", showHidden);
         console.log("filterText", filterText);
@@ -192,8 +214,11 @@ export class SearchParams {
 
 // Statics
 
-SearchParams.pageSize = config?.caching?.pageSize || 50;
+SearchParams.pageSize = 30;
 
 SearchParams.equals = function (a, b) {
-    return deepEqual(a.export(), b.export());
+    if (a !== undefined && b !== undefined) {
+        return deepEqual(a.export(), b.export());
+    }
+    return false;
 };

client/src/components/History/model/index.js

@@ -5,6 +5,7 @@ export { JobStateSummary } from "./JobStateSummary";
 export { DateStore } from "./DateStore";
 export { History } from "./History";
 export { SearchParams } from "./SearchParams";
+export { ScrollPos } from "./ScrollPos";
 export { CurveFit } from "./CurveFit";
 export { STATES } from "./states";
 

client/src/components/History/providers/CollectionContentProvider/CollectionContentProvider.js

@@ -1,21 +1,6 @@
-/**
- * Same basic format as historyContentProvider, but it should be simpler than
- * the history because:
- *
- * 1. Collections are immutable after creation so they don't change over time,
- *    meaning there is no polling and no need to do "random access" as we do for
- *    histories since we can just limit/offset to reach everything since the
- *    total matches count will always be the same. A "total matches" exists in
- *    the basic content record in the form of "element_count", and we already
- *    have that value in the DSC which is passed in.
- *
- * 2. There is no filtering for collections on the server so all we can do is
- *    page through the results.
- */
-
 import { DatasetCollection } from "../../model";
-import { ContentProvider } from "../ContentProvider";
-import { processCollectionStreams } from "./processCollectionStreams";
+import { ContentProvider, processContentStreams } from "../ContentProvider";
+import { collectionPayload } from "./collectionPayload";
 
 export default {
     mixins: [ContentProvider],
@@ -39,11 +24,11 @@ export default {
 
     methods: {
         initStreams() {
-            const { debouncePeriod, pageSize, params$, scrollPos$ } = this;
-            const dsc$ = this.watch$("dsc", true);
-            const sources = { params$, dsc$, scrollPos$ };
-            const settings = { debouncePeriod, pageSize };
-            return processCollectionStreams(sources, settings);
+            const { debouncePeriod, pageSize, params$, scrollPos$, debug } = this;
+            const parent$ = this.watch$("dsc", true);
+            const sources = { params$, parent$, scrollPos$ };
+            const settings = { debouncePeriod, pageSize, debug };
+            return processContentStreams(collectionPayload, sources, settings);
         },
     },
 };

client/src/components/History/providers/CollectionContentProvider/CollectionContentProvider.test.js

@@ -1,172 +1,154 @@
-import { NEVER } from "rxjs";
-import { cacheContent, cacheCollectionContent, bulkCacheDscContent, wipeDatabase } from "../../caching";
+import { map } from "rxjs/operators";
+import { watchUntil, getLocalVue } from "jest/helpers";
+import { shallowMount } from "@vue/test-utils";
+import { cacheContent, cacheCollectionContent, wipeDatabase, loadDscContent } from "../../caching";
+import { bulkCacheDscContent } from "../../caching/db/observables";
+import { summarizeCacheOperation } from "../../caching/loadHistoryContents";
 import { SearchParams } from "../../model/SearchParams";
 import CollectionContentProvider from "./CollectionContentProvider";
-import { watchForChange, mountRenderless, getLocalVue } from "jest/helpers";
-
-// sample data
-import rawRootCollection from "../../test/json/DatasetCollection.json";
-import rawCollection from "../../test/json/collectionContent.json";
-import rawNestedCollection from "../../test/json/collectionContent.nested.json";
-
-//#region Mocking
+import { sameVueObj, payloadChange } from "../../test/providerTestHelpers";
+import { testCollection, testCollectionContent } from "../../test/testCollection";
+import { defaultPayload } from "../ContentProvider";
+// import { reportPayload } from "../../test/providerTestHelpers";
 
+// mocking
 jest.mock("app");
-
 jest.mock("../../caching");
 
-import { loadCollectionContents } from "./loadCollectionContents";
-jest.mock("./loadCollectionContents");
-
-// Don't actually load anything, we're testing the cache response
-loadCollectionContents.mockImplementation((config) => (src$) => {
-    return NEVER;
+// fake server call
+// prettier-ignore
+loadDscContent.mockImplementation((cfg) => (inputs$) => {
+    return inputs$.pipe(
+        map((inputs) => {
+            const [url, , pagination] = inputs;
+            const { offset, limit } = pagination;
+            // console.log("mock loader", url, pagination);
+            const result = testCollectionContent
+                .filter((row) => url == row.parent_url)
+                .slice(offset, offset + limit);
+            return result;
+        }),
+        bulkCacheDscContent(),
+        summarizeCacheOperation()
+    );
 });
 
-//#endregion
+let dsc;
+const localVue = getLocalVue();
 
-const mountProvider = async (Component, propsData) => {
-    const localVue = getLocalVue();
-    const wrapper = mountRenderless(Component, { localVue, propsData });
-    await wrapper.vm.$nextTick();
-    return wrapper;
-};
+beforeEach(async () => {
+    dsc = await cacheContent(testCollection, true);
+});
+
+afterEach(async () => {
+    dsc = null;
+    await wipeDatabase();
+});
 
 describe("CollectionContentProvider", () => {
-    beforeEach(wipeDatabase);
-    afterEach(wipeDatabase);
-
+    const pageSize = 5;
+    const debouncePeriod = 250;
     let wrapper;
-    afterEach(() => {
+
+    afterEach(async () => {
         if (wrapper) wrapper.destroy();
+        wrapper = undefined;
     });
 
-    describe("initialization state", () => {
-        let parent;
-
-        // No loading
+    describe("blank initialization state", () => {
+        // current payload
         beforeEach(async () => {
-            parent = await cacheContent(rawRootCollection, true);
-            wrapper = await mountProvider(CollectionContentProvider, {
-                parent,
-                disableLoad: true,
+            wrapper = await shallowMount(CollectionContentProvider, {
+                localVue,
+                propsData: {
+                    parent: dsc,
+                    pageSize,
+                    debug: false,
+                    debouncePeriod,
+                },
+                scopedSlots: {
+                    default() {},
+                },
             });
+            await wrapper.vm.$nextTick();
         });
 
-        test("should load with default values", async () => {
-            await watchForChange({ vm: wrapper.vm, propName: "payload" });
-            const { contents, startKey, topRows, bottomRows, totalMatches } = wrapper.vm.payload;
-
-            // no content, so no startKey and empty contnets
-            expect(contents).toEqual([]);
-            expect(startKey).toEqual(undefined);
-
-            // result derived entirely from parent
-            expect(topRows).toEqual(0);
-            expect(bottomRows).toEqual(parent.element_count);
-            expect(totalMatches).toEqual(parent.element_count);
-
-            // check passthrough vals
-            expect(wrapper.vm.params).toEqual(new SearchParams());
-            expect(wrapper.vm.pageSize).toEqual(SearchParams.pageSize);
-
-            // check exposed methods
-            expect(wrapper.vm.updateParams).toBeInstanceOf(Function);
+        test("should expose update methods", () => {
+            // allows child scroller component to update the position in the history
+            // via an event handler
             expect(wrapper.vm.setScrollPos).toBeInstanceOf(Function);
         });
+
+        test("should show an empty payload", () => {
+            expect(SearchParams.equals(new SearchParams(), wrapper.vm.params)).toBe(true);
+            expect(sameVueObj(wrapper.vm.payload, defaultPayload));
+        });
     });
 
-    describe("should emit payload to reflect cached content", () => {
-        let parent;
-        let children;
-        let grandChildren;
-
-        // dummy up children to have matching ids
-        const loadChildren = async (childProps, parent_url, returnDocs) => {
-            const processed = childProps.map((props) => ({ parent_url, ...props }));
-            return await bulkCacheDscContent(processed, returnDocs);
-        };
-
-        const loadCollectionTree = async () => {
-            // root dsc
-            parent = await cacheContent(rawRootCollection, true);
-
-            // first layer of collection contents
-            children = await loadChildren(rawCollection, parent.contents_url, true);
-
-            // 2nd layer of nested collection contents
-            grandChildren = [];
-            for (const child of children) {
-                const gc = await loadChildren(rawNestedCollection, child.contents_url, true);
-                grandChildren.push(...gc);
-            }
-        };
-
-        // No loading
+    describe("waiting for simulated server loads", () => {
         beforeEach(async () => {
-            await loadCollectionTree();
-            parent = await cacheContent(rawRootCollection, true);
-            wrapper = await mountProvider(CollectionContentProvider, {
-                parent,
-                disableLoad: true,
+            wrapper = await shallowMount(CollectionContentProvider, {
+                localVue,
+                propsData: {
+                    parent: dsc,
+                    pageSize,
+                    debug: false,
+                    debouncePeriod,
+                },
+                scopedSlots: {
+                    default: function (props) {
+                        // reportPayload(props.payload, { indexKey: "element_index" });
+                        // payload = props.payload;
+                    },
+                },
             });
-        });
 
-        test("should load with pre-inserted cached children values", async () => {
-            await watchForChange({ vm: wrapper.vm, propName: "payload" });
-            const { contents, startKey, topRows, bottomRows, totalMatches } = wrapper.vm.payload;
-            const { params, pageSize } = wrapper.vm;
-
-            const ids = contents.map((o) => o.id);
-            const childIds = children.map((o) => o.id);
-
-            expect(ids).toEqual(childIds);
-            expect(startKey).toBe(0);
-            expect(topRows).toBe(0);
-            expect(bottomRows).toBe(0); // length of all children < pagesize
-            expect(totalMatches).toBe(children.length);
-            expect(params).toEqual(new SearchParams());
-            expect(pageSize).toEqual(SearchParams.pageSize);
-        });
-
-        // If the cache adds some content, should still be at the top, but the
-        // bottomRows should change to reflect the new rows that are now
-        // rendered in the contents
-
-        test("should emit when a new item is inserted", async () => {
-            await watchForChange({ vm: wrapper.vm, propName: "payload" });
-            expect(wrapper.vm.payload.contents.length).toEqual(children.length);
-
-            // clone and update a row
-            const lastChild = children[children.length - 1];
             // eslint-disable-next-line no-unused-vars
-            const { _id, _rev, ...existingProps } = lastChild;
-            const newChild = {
-                ...existingProps,
-                element_index: existingProps.element_index + 1,
-            };
+            const spinUp = await watchUntil(wrapper.vm, function () {
+                const allDone = wrapper.vm.payload.contents.length > 0;
+                // console.log("done?", allDone);
+                return allDone;
+            });
 
-            const insertResult = await cacheCollectionContent(newChild);
-            expect(insertResult.updated).toBe(true);
-            await watchForChange({ vm: wrapper.vm, propName: "payload" });
-
-            const { contents } = wrapper.vm.payload;
-            expect(contents.length).toBe(children.length + 1);
+            // console.log(spinUp);
         });
 
-        test("should emit when an existing item is updated", async () => {
-            await watchForChange({ vm: wrapper.vm, propName: "payload" });
+        test("initial load, no scrolling", async () => {
+            const payload = wrapper.vm.payload;
+            expect(payload).toBeDefined();
+            // reportPayload(payload, { indexKey: "element_index" });
+            // reportPayload(initialPayload, { indexKey: "element_index" });
+            const { contents, topRows, bottomRows, startKey, totalMatches } = payload;
+            expect(topRows).toEqual(0);
+            expect(contents.length).toBeGreaterThanOrEqual(2 * wrapper.vm.pageSize);
+            expect(totalMatches).toEqual(testCollectionContent.length);
+            expect(bottomRows).toBeGreaterThan(0);
+            expect(bottomRows).toEqual(testCollectionContent.length - contents.length);
 
-            const lastChild = children[children.length - 1];
-            lastChild.foo = 123;
+            const testStartIndex = testCollectionContent[0].element_index;
+            expect(startKey).toEqual(testStartIndex);
+        });
 
-            const updatedResult = await cacheCollectionContent(lastChild);
-            expect(updatedResult.updated).toBe(true);
+        test("should update to reflect independent cache changes", async () => {
+            let hasFoobar;
+            const payload = wrapper.vm.payload;
 
-            await watchForChange({ vm: wrapper.vm, propName: "payload" });
+            // check initial emit, for foobars
+            expect(payload).toBeDefined();
+            hasFoobar = payload.contents.some((o) => o.foobar == 123);
+            expect(hasFoobar).toBeFalsy();
 
-            const { contents } = wrapper.vm.payload;
-            expect(contents.length).toBe(children.length);
+            // add a foobar
+            const testChild = payload.contents[0];
+            testChild.foobar = 123;
+            const updateResult = await cacheCollectionContent(testChild, true);
+            expect(updateResult.foobar).toEqual(123);
+
+            // check subsequent payload for a foobar
+            const updatedPayload = await payloadChange({ vm: wrapper.vm });
+            // reportPayload(updatedPayload, { label: "updated", indexKey: "element_index" });
+            hasFoobar = updatedPayload.contents.some((o) => o.foobar == 123);
+            expect(hasFoobar).toBeTruthy();
         });
     });
 });

client/src/components/History/providers/CollectionContentProvider/collectionPayload.js

@@ -0,0 +1,69 @@
+import { of, Observable, BehaviorSubject } from "rxjs";
+import { tap, map, pluck, switchMap, share, debounceTime } from "rxjs/operators";
+import { chunkProp, show } from "utils/observable";
+import { SearchParams } from "../../model/SearchParams";
+import { buildPaginationWindow } from "../ContentProvider";
+import { loadDscContent } from "../../caching";
+import { watchCollection } from "./watchCollection";
+// import { reportPayload } from "../../test/providerTestHelpers";
+
+// prettier-ignore
+export const collectionPayload = (cfg = {}) => pos$ => {
+    const {
+        parent: dsc,
+        filters = new SearchParams(),
+        totalMatches = dsc.totalElements,
+        pageSize = SearchParams.pageSize,
+        debouncePeriod = 250,
+        debug = false,
+        loading$ = new BehaviorSubject(),
+    } = cfg;
+
+    const { contents_url } = dsc;
+
+    const cursor$ = pos$.pipe(
+        pluck('cursor'),
+        show(debug, cursor => console.log("collectionPayload input cursor", cursor)),
+        share()
+    );
+
+    // Server loads
+
+    const pagination$ =  cursor$.pipe(
+        map((cursor) => buildPaginationWindow(cursor, totalMatches, pageSize)),
+        chunkProp('offset', pageSize / 2),
+        show(debug, pagination => console.log("collectionPayload pagination", pagination)),
+    );
+
+    const serverLoad$ = pagination$.pipe(
+        tap(() => loading$.next(true)),
+        switchMap(pagination => of([contents_url, filters, pagination]).pipe(
+            loadDscContent(),
+        )),
+        show(debug, val => console.log("collectionPayload loadDscContent response", val)),
+    );
+
+    // Cache Watcher
+    // convert cursor to key: element_index, this is a lot easier than on
+    // history because we don't filter (yet) and the index is numeric starting
+    // at the top, so all we have to do is scale it
+    const payload$ = cursor$.pipe(
+        map((cursor) => cursor * totalMatches),
+        watchCollection({ dsc, filters, ...cfg}),
+        debounceTime(debouncePeriod),
+        map((result) => {
+            const { contents } = result;
+            const topRows = contents.length ? contents[0].element_index : 0;
+            const bottomRows = Math.max(0, totalMatches - contents.length - topRows);
+            return { ...result, topRows, bottomRows, totalMatches };
+        }),
+    );
+
+    // create observable that looks at the cache but piggybacks the loader subscription
+    // piggyback the subscriptions
+    return new Observable((obs) => {
+        const watchSub = payload$.subscribe(obs);
+        watchSub.add(serverLoad$.subscribe());
+        return watchSub;
+    })
+};

client/src/components/History/providers/CollectionContentProvider/index.js

@@ -1,3 +1,18 @@
+/**
+ * Same basic format as historyContentProvider, but it should be simpler than
+ * the history because:
+ *
+ * 1. Collections are immutable after creation so they don't change over time,
+ *    meaning there is no polling and no need to do "random access" as we do for
+ *    histories since we can just limit/offset to reach everything since the
+ *    total matches count will always be the same. A "total matches" exists in
+ *    the basic content record in the form of "element_count", and we already
+ *    have that value in the DSC which is passed in.
+ *
+ * 2. There is no filtering for collections on the server so all we can do is
+ *    page through the results
+ */
+
 import CollectionContentProvider from "./CollectionContentProvider";
 
 export default CollectionContentProvider;

client/src/components/History/providers/CollectionContentProvider/loadCollectionContents.js

@@ -1,17 +0,0 @@
-import { of } from "rxjs";
-import { switchMap } from "rxjs/operators";
-import { tag } from "rxjs-spy/operators/tag";
-import { loadDscContent } from "../../caching";
-
-export const loadCollectionContents = (cfg = {}) => {
-    const { windowSize = 100 } = cfg;
-
-    // prettier-ignore
-    return switchMap(([dsc, params, idx]) => {
-        const { contents_url } = dsc;
-        return of([contents_url, params, idx]).pipe(
-            tag("ajaxLoadInputs"), 
-            loadDscContent({ windowSize })
-        );
-    });
-};

client/src/components/History/providers/CollectionContentProvider/processCollectionStreams.js

@@ -1,102 +0,0 @@
-import { merge, combineLatest } from "rxjs";
-import { map, pluck, distinctUntilChanged, share, debounceTime } from "rxjs/operators";
-import { tag } from "rxjs-spy/operators/tag";
-import { whenAny } from "utils/observable/whenAny";
-import { activity } from "utils/observable/activity";
-import { shareButDie } from "utils/observable/shareButDie";
-import { chunk } from "../../caching/operators/chunk";
-
-import { DatasetCollection, SearchParams } from "../../model";
-import { watchCollectionCache } from "./watchCollectionCache";
-import { loadCollectionContents } from "./loadCollectionContents";
-import { loadInputsSame } from "../HistoryContentProvider/processHistoryStreams";
-
-// prettier-ignore
-export function processCollectionStreams(sources, settings) {
-    const { debouncePeriod, pageSize } = settings;
-    const { dsc$: rawDsc$, params$: rawParams$, scrollPos$ } = sources;
-
-    //#region Clean up, filter, debounce raw inputs
-
-    const dsc$ = rawDsc$.pipe(distinctUntilChanged(DatasetCollection.equals));
-    const params$ = rawParams$.pipe(distinctUntilChanged(SearchParams.equals));
-    const inputs$ = whenAny(dsc$, params$).pipe(tag('inputs'), share());
-    const scrolling$ = scrollPos$.pipe(activity());
-    const totalMatches$ = dsc$.pipe(pluck("totalElements"));
-
-    //#endregion
-
-    //#region Estimate element index from cursor + dsc
-
-    const cursor$ = whenAny(scrollPos$, totalMatches$).pipe(
-        map(([pos, totalMatches]) => getIndexFromPos(pos, totalMatches)),
-        distinctUntilChanged(),
-        tag('cursor'),
-        shareButDie(1),
-    );
-
-    //#endregion
-
-    //#region Loading
-
-    const chunkedCursor$ = cursor$.pipe(
-        chunk(pageSize), 
-    );
-
-    const loadInputs$ = whenAny(inputs$, chunkedCursor$).pipe(
-        distinctUntilChanged(loadInputsSame),
-        map(([inputs, idx]) => [...inputs, idx]),
-        tag('loadInputs'),
-        shareButDie(1)
-    );
-
-    const loadResult$ = loadInputs$.pipe(
-        loadCollectionContents({ windowSize: 2 * pageSize }),
-        tag('loadResult'),
-    );
-
-    const loading$ = merge(loadInputs$, loadResult$).pipe(
-        activity(),
-    );
-
-    //#endregion
-
-    //#region Cache watcher
-
-    const cacheFromMonitor$ = inputs$.pipe(
-        watchCollectionCache({
-            cursor$,
-            pageSize,
-            debouncePeriod,
-        }),
-        tag('cacheFromMonitor'),
-    );
-
-    const payload$ = combineLatest(cacheFromMonitor$, totalMatches$).pipe(
-        debounceTime(debouncePeriod),
-        map((inputs) => buildPayload(...inputs)),
-        tag('payload'),
-    );
-
-    //#endregion
-
-    return { payload$, scrolling$, loading$ };
-}
-
-function getIndexFromPos(pos, totalMatches) {
-    const { cursor = null, key = null } = pos;
-    if (key !== null) {
-        return key;
-    }
-    if (cursor !== null) {
-        return Math.round(Number(cursor) * totalMatches);
-    }
-    return 0;
-}
-
-function buildPayload(result, totalMatches) {
-    const { contents } = result;
-    const topRows = contents.length ? contents[0].element_index : 0;
-    const bottomRows = Math.max(0, totalMatches - contents.length - topRows);
-    return { ...result, topRows, bottomRows, totalMatches };
-}

client/src/components/History/providers/CollectionContentProvider/watchCollection.js

@@ -0,0 +1,38 @@
+import { of } from "rxjs";
+import { map } from "rxjs/operators";
+import { aggregateCacheUpdates } from "../ContentProvider";
+import { monitorCollectionContent } from "../../caching";
+import { SEEK } from "../../caching/enums";
+import { SearchParams } from "../../model";
+// import { show } from "utils/observable";
+
+// prettier-ignore
+export const watchCollection = (cfg = {}) => elementIndex$ => {
+    const { 
+        dsc, 
+        filters = new SearchParams(),
+        pageSize = SearchParams.pageSize, 
+        debug = false,
+        keyField = "element_index",
+        keyDirection = SEEK.ASC,
+        ...otherConfig 
+    } = cfg;
+
+    // builds a monitor observable based on a element_index
+    const monitor = (idx) => of([dsc.contents_url, filters, idx]).pipe(
+        monitorCollectionContent({ pageSize, debug, keyField }),
+    );
+    
+    // handles the plumbing for observing the query (monitor) and collecting the output into an
+    // aggregate ordered list focused around the current target key (element_index in this case)
+    return elementIndex$.pipe(
+        map(val => parseInt(val)),
+        aggregateCacheUpdates(monitor, {
+            keyField,
+            keyDirection,
+            pageSize, 
+            debug,
+            ...otherConfig,
+        }),
+    );
+};

client/src/components/History/providers/CollectionContentProvider/watchCollection.test.js

@@ -0,0 +1,125 @@
+import { of, timer } from "rxjs";
+import { takeUntil, share } from "rxjs/operators";
+import { ObserverSpy } from "@hirez_io/observer-spy";
+import { watchCollection } from "./watchCollection";
+import { cacheCollectionContent, bulkCacheDscContent, wipeDatabase } from "../../caching";
+import { testCollectionContent, testCollection } from "../../test/testCollection";
+import { untilNthEmission } from "jest/helpers";
+
+// need to mock monitorHistoryContent which is instantiated inside the worker,
+// but we can't actually fire up threads.js in a unit test because you can't
+// call expose() on the main thread, so we mock the adapter functions which
+// generate observables from the worker.
+jest.mock("app");
+jest.mock("../../caching");
+
+const debouncePeriod = 250;
+const safetyTimeout = 1000;
+const pageSize = 5;
+const pluckAll = (arr, prop) => arr.map((o) => o[prop]);
+
+let cachedCollectionContent;
+
+beforeEach(async () => {
+    await wipeDatabase();
+    cachedCollectionContent = await bulkCacheDscContent(testCollectionContent, true);
+    // const cachedIds = pluckAll(cachedCollectionContent, "_id");
+    // console.log(cachedIds);
+});
+
+afterEach(async () => {
+    await wipeDatabase();
+});
+
+describe("watchCollection", () => {
+    test("should observe pre-inserted content on the initial emission", async () => {
+        // the srource stream for the operator is an element_index
+        const watcher$ = of(0).pipe(
+            watchCollection({
+                dsc: testCollection,
+                pageSize,
+                debouncePeriod,
+                debug: false,
+            }),
+            takeUntil(timer(safetyTimeout))
+        );
+
+        // spy on output of observable, wait for it to end
+        const spy = new ObserverSpy();
+        watcher$.subscribe(spy);
+        await spy.onComplete();
+
+        expect(spy.receivedNext()).toBe(true);
+        expect(spy.receivedComplete()).toBe(true);
+        expect(spy.getValuesLength()).toBe(1);
+
+        const payload = spy.getFirstValue();
+        {
+            const { startKey, startKeyIndex, targetKey, contents } = payload;
+
+            // at the top of the list, should get the match (element_index = 0) + 2 pages
+            expect(startKey).toEqual(0);
+            expect(startKeyIndex).toEqual(0);
+            expect(targetKey).toEqual(0);
+            expect(Array.isArray(contents)).toBe(true);
+
+            // should get back the exact match (element_index = 0, the following page, + 1
+            // additional page of buffer-room after the current page
+            const returnedIndexes = pluckAll(contents, "element_index");
+            // console.log(returnedIndexes);
+
+            expect(contents.length).toBeGreaterThanOrEqual(2 * pageSize);
+            expect(returnedIndexes).toEqual([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]);
+        }
+    });
+
+    test("should observe subsequent updates", async () => {
+        // the srource stream for the operator is an element_index
+        const watcher$ = of(0).pipe(
+            watchCollection({
+                dsc: testCollection,
+                debouncePeriod,
+                pageSize,
+            }),
+            takeUntil(timer(safetyTimeout)),
+            share()
+        );
+
+        const spy = new ObserverSpy();
+        watcher$.subscribe(spy);
+
+        // After the first event, change some stuff on the first row
+        // need to delay a little to wait for first emissions to run through
+        // otherwise the debouncing will think the 2nd update is the same as the
+        await untilNthEmission(watcher$, 1);
+        const testChild = cachedCollectionContent[0];
+        testChild.foobar = 123;
+        const updateResult = await cacheCollectionContent(testChild, true);
+        // console.log("after update", updateResult);
+        expect(updateResult.foobar).toEqual(123);
+        expect(updateResult._rev).not.toEqual(testCollection._rev);
+
+        await spy.onComplete();
+        expect(spy.receivedNext()).toBe(true);
+        expect(spy.receivedComplete()).toBe(true);
+        expect(spy.getValuesLength()).toEqual(2);
+
+        const firstPayload = spy.getFirstValue();
+        {
+            const { contents } = firstPayload;
+            expect(Array.isArray(contents)).toBe(true);
+            // no foobar because this happened before the update
+            const foobars = pluckAll(contents, "foobar");
+            expect(foobars.every((val) => val === undefined)).toBe(true);
+        }
+
+        const lastPayload = spy.getLastValue();
+        {
+            const { contents } = lastPayload;
+            expect(Array.isArray(contents)).toBe(true);
+            // should have a foobar
+            const foobars = pluckAll(contents, "foobar");
+            expect(foobars.every((val) => val === undefined)).toBe(false);
+        }
+    });
+});

client/src/components/History/providers/CollectionContentProvider/watchCollectionCache.js

@@ -1,96 +0,0 @@
-import { combineLatest } from "rxjs";
-import { map, switchMap, scan, distinctUntilChanged, catchError } from "rxjs/operators";
-import { tag } from "rxjs-spy/operators/tag";
-import { chunk } from "../../caching/operators/chunk";
-import { monitorDscQuery } from "../../caching";
-import { processContentUpdate, newUpdateMap, buildContentResult, getKeyForUpdateMap } from "../aggregation";
-import { SearchParams } from "../../model";
-
-// prettier-ignore
-export const watchCollectionCache = (cfg = {}) => input$ => {
-    const {
-        cursor$,
-        pageSize = SearchParams.pageSize,
-        monitorChunk = 5 * pageSize,
-        monitorPageSize = 5 * monitorChunk,
-        inputDebounce = 250,
-        // outputDebounce = inputDebounce,
-        keyField = "element_index",
-        keyDirection = "asc",
-    } = cfg;
-
-    const getKey = getKeyForUpdateMap(keyField);
-    const aggregator = processContentUpdate({ getKey });
-    const summarize = buildContentResult({ pageSize, keyDirection, getKey });
-
-    const contentMap$ = input$.pipe(
-        switchMap(([{contents_url}, params]) => {
-
-            const chunkedCursor$ = cursor$.pipe(
-                chunk(monitorChunk, true),
-                distinctUntilChanged(),
-            );
-
-            const monitorInput$ = chunkedCursor$.pipe(
-                map((idx) => [contents_url, params, idx]),
-                tag('monitorInputs'),
-            );
-
-            const request$ = monitorInput$.pipe(
-                map(buildDscChildrenSelector(monitorPageSize)),
-                tag('request'),
-            );
-
-            const monitorOutput$ = request$.pipe(
-                monitorDscQuery({ inputDebounce }),
-                tag('monitorOutput'),
-            );
-
-            // aggregate results in a map for each set of id + params
-            const updateMap$ = monitorOutput$.pipe(
-                scan(aggregator, newUpdateMap()),
-            );
-
-            return updateMap$;
-        }),
-    );
-
-    const contentWindow$ = combineLatest([ contentMap$, cursor$ ]).pipe(
-        // debounceTime(outputDebounce),
-        map(summarize),
-        tag('contentWindow'),
-    );
-
-    return contentWindow$.pipe(
-        catchError(err => {
-            console.warn("Error in watchHistoryContents", err);
-            throw err;
-        })
-    );
-};
-
-// filters currently unused, but they could be if we add some filter UI
-export const buildDscChildrenSelector = (pageSize) => (inputs) => {
-    // console.log("buildDscChildrenSelector", pageSize, inputs);
-
-    // eslint-disable-next-line no-unused-vars
-    const [parent_url, filters, cursor] = inputs;
-    const lowerLimit = Math.max(0, cursor - pageSize);
-
-    const request = {
-        selector: {
-            _id: { $gte: `${parent_url}-${lowerLimit}` },
-            parent_url: { $eq: parent_url },
-        },
-        sort: [{ _id: "asc" }],
-        limit: pageSize,
-        index: {
-            fields: ["_id"],
-            sort: [{ _id: "asc" }],
-            name: "collection contents by parent_url and element_index ascending",
-            ddoc: "idx-dsc-contents-by-url-and-element-asc",
-        },
-    };
-
-    return request;
-};

client/src/components/History/providers/CollectionContentProvider/watchCollectionContents.test.js

@@ -1,114 +0,0 @@
-import { of, isObservable, timer } from "rxjs";
-import { take, takeUntil, share } from "rxjs/operators";
-import { ObserverSpy } from "@hirez_io/observer-spy";
-// import { wait } from "jest/helpers";
-import { SearchParams } from "../../model/SearchParams";
-import { watchCollectionCache } from "./watchCollectionCache";
-
-import { cacheCollectionContent, bulkCacheDscContent, wipeDatabase } from "../../caching";
-
-import rawCollection from "../../test/json/collectionContent.json";
-
-jest.mock("app");
-
-// need to mock monitorHistoryContent which is instantiated inside the worker,
-// but we can't actually fire up threads.js in a unit test because you can't
-// call expose() on the main thread, so we mock the adapter functions which
-// generate observables from the worker.
-jest.mock("../../caching");
-
-beforeEach(wipeDatabase);
-afterEach(wipeDatabase);
-
-// prettier-ignore
-describe("watchCollectionContents", () => {
-    describe("single layer collection", () => {
-        let children;
-
-        const testTimeout = 3000;
-        const parent_url = "/foo";
-
-        const cursor$ = of(0);
-        const fakeDscParent = { contents_url: parent_url };
-        const input$ = of([fakeDscParent, new SearchParams()]);
-
-        // insert a list of collection contents
-        beforeEach(async () => {
-            const rawChildren = rawCollection.map((props) => ({ ...props, parent_url }));
-            children = await bulkCacheDscContent(rawChildren, true);
-        });
-
-        test("sanity check inputs", () => {
-            expect(children).toBeDefined();
-            expect(Array.isArray(children)).toBe(true);
-            expect(children.length).toEqual(rawCollection.length);
-            expect(children.every((doc) => doc.parent_url == parent_url)).toBe(true);
-        });
-
-        test("should observe pre-inserted collection content", async () => {
-            const watcher$ = input$.pipe(
-                watchCollectionCache({ cursor$, pageSize: 5 }),
-                take(1),
-                takeUntil(timer(testTimeout))
-            );
-            expect(isObservable(watcher$)).toBe(true);
-
-            // spy on output of observable, wait for it to end
-            const spy = new ObserverSpy();
-            watcher$.subscribe(spy);
-            await spy.onComplete();
-
-            expect(spy.receivedNext()).toBe(true);
-            expect(spy.receivedComplete()).toBe(true);
-
-            const { startKey, contents } = spy.getFirstValue();
-            expect(startKey).toEqual(0);
-            expect(Array.isArray(contents)).toEqual(true);
-            expect(contents.length).toEqual(rawCollection.length);
-            // show(contents);
-        });
-
-        test("should observe subsequent updates", async () => {
-
-            const watcher$ = input$.pipe(
-                watchCollectionCache({
-                    cursor$,
-                    pageSize: 100,
-                    inputDebounce: 0,
-                    outputDebounce: 100,
-                }),
-                take(2),
-                takeUntil(timer(2000)),
-                share()
-            );
-            expect(isObservable(watcher$)).toBe(true);
-
-            // After the first event, change some stuff
-
-            const firstWatcherEvent$ = watcher$.pipe(take(1));
-
-            firstWatcherEvent$.subscribe(async () => {
-                // Update one of the rows
-                const testCollection = children[0];
-                testCollection.foobar = 123;
-                const updateResult = await cacheCollectionContent(testCollection, true);
-                expect(updateResult.foobar).toEqual(123);
-                expect(updateResult._rev).not.toEqual(testCollection._rev);
-            });
-
-            // spy on output of observable, wait for it to end
-            const spy = new ObserverSpy();
-            watcher$.subscribe(spy);
-            await spy.onComplete();
-
-            expect(spy.receivedNext()).toBe(true);
-            expect(spy.receivedComplete()).toBe(true);
-            expect(spy.getValuesLength()).toEqual(2);
-
-            const { contents } = spy.getLastValue();
-            expect(Array.isArray(contents)).toBe(true);
-            expect(contents.length).toEqual(children.length);
-            expect(contents[0].foobar).toEqual(123);
-        });
-    });
-});

client/src/components/History/providers/ContentProvider.js

@@ -1,159 +0,0 @@
-import { NEVER, BehaviorSubject } from "rxjs";
-import { SearchParams } from "../model/SearchParams";
-
-// dumb math util
-export const clamp = (val, [bottom, top]) => Math.max(bottom, Math.min(top, val));
-
-// simple comparators
-export const isDefined = (val) => val !== null && val !== undefined;
-
-// defined, number and finite
-export const isValidNumber = (val) => {
-    return isDefined(val) && !isNaN(val) && isFinite(val);
-};
-
-// validate output variables before rendering
-export const validPayload = ({ topRows, bottomRows, totalMatches }) => {
-    return isValidNumber(topRows) && isValidNumber(bottomRows) && isValidNumber(totalMatches);
-};
-
-// comparator for distinct() style operators inputs are an array of [id, SearchParams]
-export const inputsSame = ([a0, a1], [b0, b1]) => {
-    return a0 == b0 && SearchParams.equals(a1, b1);
-};
-
-export const scrollPosEquals = (a, b) => {
-    return a.cursor === b.cursor && a.key === b.key;
-};
-
-export const ContentProvider = {
-    props: {
-        parent: { type: Object, required: true },
-        pageSize: { type: Number, default: SearchParams.pageSize },
-        debouncePeriod: { type: Number, default: 100 },
-        disableWatch: { type: Boolean, default: false },
-        disableLoad: { type: Boolean, default: false },
-        debug: { type: Boolean, default: false },
-    },
-
-    computed: {
-        busy() {
-            return this.loading || this.scrolling;
-        },
-    },
-
-    data() {
-        return {
-            payload: {},
-            params: new SearchParams(),
-            loading: false,
-            scrolling: false,
-        };
-    },
-
-    watch: {
-        params(newParams, oldParams) {
-            if (!SearchParams.equals(newParams, oldParams)) {
-                this.resetScrollPos();
-            }
-        },
-    },
-
-    created() {
-        this.initParams();
-        this.initScrollPos();
-
-        const { payload$, loading$, scrolling$ } = this.initStreams();
-
-        this.listenTo(scrolling$, (val) => (this.scrolling = val));
-        this.listenTo(loading$, (val) => (this.loading = val));
-
-        // render output
-        if (!this.disableWatch) {
-            this.listenTo(payload$, {
-                next: (payload) => this.setPayload(payload),
-                error: (err) => console.warn("error in cache$", err),
-                complete: () => console.warn("why did cache$ complete?"),
-            });
-        }
-    },
-
-    methods: {
-        initParams() {
-            this.params$ = new BehaviorSubject(new SearchParams());
-            this.listenTo(this.params$, (val) => (this.params = val));
-        },
-
-        initScrollPos() {
-            this.scrollPos$ = new BehaviorSubject({ cursor: 0.0, key: null });
-        },
-
-        resetScrollPos() {
-            this.scrollPos$.next({ cursor: 0.0, key: null });
-        },
-
-        initStreams() {
-            console.warn("Override initStreams in ContentProvider");
-            return { cache$: NEVER, loader$: NEVER, loading$: NEVER, scrolling$: NEVER };
-        },
-
-        /**
-         * Handler for the scroller to update its position, either by key or
-         * cursor (0-1 value representing how far down it is)
-         * @param {object} Object consisting of key and cursor props
-         */
-        setScrollPos({ cursor = 0.0, key = null } = {}) {
-            if (isValidNumber(cursor)) {
-                this.scrollPos$.next({ cursor, key });
-            }
-        },
-
-        /**
-         * After bulk operations there is a need to trigger a fresh load.
-         */
-        manualReload() {
-            this.setScrollPos(this.scrollPos$.value);
-        },
-
-        /**
-         * Exposed method so child components can update the search parameters.
-         * @param {SearchParams} newParams Fresh search params
-         */
-        updateParams(newParams) {
-            const val = newParams instanceof SearchParams ? newParams.clone() : new SearchParams(newParams);
-            this.params$.next(val);
-        },
-
-        /**
-         * Render cache observable results to the payload property. It's best to
-         * set everything at once to avoid multiple render passes.
-         * @param {object} result Cache observable response
-         */
-        setPayload(newPayload = {}) {
-            this.$set(this, "payload", newPayload);
-        },
-    },
-
-    render() {
-        return this.$scopedSlots.default({
-            // actual content delivery
-            payload: this.payload,
-
-            // local vars/settings/props passthrough
-            loading: this.loading,
-            scrolling: this.scrolling,
-            busy: this.busy,
-            params: this.params,
-            pageSize: this.pageSize,
-
-            // update methods
-            handlers: {
-                updateParams: this.updateParams,
-                setScrollPos: this.setScrollPos,
-                manualReload: this.manualReload,
-            },
-        });
-    },
-};
-
-export default ContentProvider;

client/src/components/History/providers/ContentProvider/ContentProvider.js

@@ -0,0 +1,127 @@
+import { NEVER } from "rxjs";
+import { SearchParams, ScrollPos } from "../../model";
+import { isValidNumber } from "./helpers";
+// import { reportPayload } from "../../test/providerTestHelpers";
+
+// first emission, emitted when parent (history) or filters changes to reset the view
+export const defaultPayload = {
+    contents: [],
+    targetKey: null,
+    startKey: null,
+    startKeyIndex: 0,
+    topRows: 0,
+    bottomRows: 0,
+    totalMatches: 0,
+};
+
+export const ContentProvider = {
+    props: {
+        parent: { type: Object, required: true },
+        params: { type: SearchParams, default: () => new SearchParams() },
+        pageSize: { type: Number, default: SearchParams.pageSize },
+        debouncePeriod: { type: Number, default: 1000 },
+        debug: { type: Boolean, default: false },
+    },
+
+    computed: {
+        busy() {
+            return this.loading || this.scrolling;
+        },
+        slotProps() {
+            return {
+                // actual content delivery
+                payload: this.payload,
+
+                // local vars/settings/props passthrough
+                scrollPos: this.scrollPos,
+                loading: this.loading,
+                scrolling: this.scrolling,
+                busy: this.busy,
+                params: this.params,
+                pageSize: this.pageSize,
+
+                // update methods
+                setScrollPos: this.setScrollPos,
+                manualReload: this.manualReload,
+            };
+        },
+    },
+
+    data() {
+        return {
+            payload: defaultPayload,
+            scrollPos: new ScrollPos(),
+            loading: false,
+            scrolling: false,
+        };
+    },
+
+    watch: {
+        params(newParams, oldParams) {
+            if (!SearchParams.equals(newParams, oldParams)) {
+                this.resetScrollPos();
+            }
+        },
+    },
+
+    created() {
+        this.params$ = this.watch$("params");
+        this.scrollPos$ = this.watch$("scrollPos");
+        const { payload$, loading$, scrolling$ } = this.initStreams();
+
+        this.listenTo(scrolling$, (val) => (this.scrolling = val));
+        this.listenTo(loading$, (val) => (this.loading = val));
+
+        // render output
+        this.listenTo(payload$, {
+            next: (payload) => this.setPayload(payload),
+            error: (err) => console.warn("error in cache$", err),
+            complete: () => console.warn("why did cache$ complete?"),
+        });
+    },
+
+    methods: {
+        resetScrollPos() {
+            this.scrollPos = ScrollPos.create();
+        },
+
+        initStreams() {
+            console.warn("Override initStreams in ContentProvider");
+            return { cache$: NEVER, loader$: NEVER, loading$: NEVER, scrolling$: NEVER };
+        },
+
+        /**
+         * Handler for the scroller to update its position, either by key or
+         * cursor (0-1 value representing how far down it is)
+         * @param {object} Object consisting of key and cursor props
+         */
+        setScrollPos({ cursor = 0.0, key = null } = {}) {
+            if (isValidNumber(cursor)) {
+                this.scrollPos = ScrollPos.create({ cursor, key });
+            }
+        },
+
+        /**
+         * After bulk operations there is a need to trigger a fresh load.
+         */
+        manualReload() {
+            this.setScrollPos(this.scrollPos$.value);
+        },
+
+        /**
+         * Render cache observable results to the payload property. It's best to
+         * set everything at once to avoid multiple render passes.
+         * @param {object} result Cache observable response
+         */
+        setPayload(props = {}) {
+            const payload = Object.assign({}, defaultPayload, props);
+            this.$set(this, "payload", payload);
+        },
+    },
+
+    render() {
+        return this.$scopedSlots.default(this.slotProps);
+    },
+};
+
+export default ContentProvider;

client/src/components/History/providers/ContentProvider/aggregateCacheUpdates.js

@@ -0,0 +1,106 @@
+// Generalized bi-directional watch and aggregator.
+// Looks at cache from starting index, then looks up and down
+
+import { timer, throwError, from } from "rxjs";
+import {
+    map,
+    debounceTime,
+    distinctUntilChanged,
+    groupBy,
+    reduce,
+    mergeMap,
+    switchMap,
+    withLatestFrom,
+} from "rxjs/operators";
+import { chunk, show, debounceBurst, shareButDie } from "utils/observable";
+import { SearchParams } from "../../model/SearchParams";
+import { processContentUpdate, newUpdateMap, buildContentResult, getKeyForUpdateMap } from "./aggregation";
+import { SEEK } from "../../caching/enums";
+import { reportPayload } from "components/History/test/providerTestHelpers";
+
+// prettier-ignore
+export const aggregateCacheUpdates = (monitor, cfg = {}) => (src$) => {
+    // monitor is a function that generates the update observable
+    if (!(monitor instanceof Function)) {
+        return throwError("monitor should be a function in aggregateCacheUpdates");
+    }
+
+    const {
+        // number of rows we roughly expect to be in the window
+        pageSize = SearchParams.pageSize,
+
+        // input and output throttling
+        debouncePeriod = 250,
+
+        // field and key order for the contents
+        keyField = "hid",
+        keyDirection = SEEK.DESC,
+
+        // interval in which we create new monitors
+        // if we chunk the inputs then we won't make as many
+        // might be easier on memory
+        inputChunk = Math.floor(pageSize / 2),
+
+        // reduces duplicate updates to the skiplist
+        // which can be expensive
+        updateGroupLifetime = 2 * debouncePeriod,
+
+        debug = false,
+    } = cfg;
+
+    // function that extracts the key from a document
+    const getKey = getKeyForUpdateMap(keyField);
+
+    // incoming target key
+    const targetKey$ = src$.pipe(shareButDie(1));
+
+    // avoids creating a new monitor for every single key emission
+    // chunk to ceiling if descending, floor if ascending
+    const monitorInputKey$ = targetKey$.pipe(
+        chunk(inputChunk, keyDirection == SEEK.DESC), 
+        distinctUntilChanged(),
+        show(debug, (key) => console.log("monitorInputKey", key)),
+    );
+
+    // incoming stream of cache updates
+    // Monitor generation function is passed in via config so we can use it against different types of queries
+    const cacheUpdates$ = monitorInputKey$.pipe(
+        switchMap(monitor),
+        show(debug, (change) => console.log("monitor output", change)),
+    );
+
+    // groups updates by skiplist key and debounces those
+    // streams individually so we're not pointlessly updating
+    // when we're just going to change it again shortly
+    const groupTimeout = () => timer(updateGroupLifetime);
+    const groupedUpdates$ = cacheUpdates$.pipe(
+        groupBy(update => update.key, undefined, groupTimeout),
+        mergeMap(updateByKey$ => updateByKey$.pipe(
+            debounceTime(0.25 * debouncePeriod),
+        )),
+        debounceBurst(debouncePeriod),
+        show(debug, changeBurst => console.log('burst', changeBurst.map(o => o.key))),
+    );
+
+    // aggregates changes into a skiplist.
+    const storage = newUpdateMap();
+    const currentMap$ = groupedUpdates$.pipe(
+        mergeMap(updateList => from(updateList).pipe(
+            reduce(processContentUpdate, storage)
+        )),
+        show(debug, () => console.log('grouped update burst done')),
+    );
+
+    const contentListInputs$ = currentMap$.pipe(
+        withLatestFrom(targetKey$),
+    );
+
+    // query skiplist with scroll position (transformed into a key from the skiplist) 
+    // to produce list of nearby content
+    const contentList$ = contentListInputs$.pipe(
+        map(buildContentResult({ pageSize, keyDirection, getKey })),
+        show(debug, payload => reportPayload(payload, { indexKey: keyField, label: "aggregateCacheUpdates payload" })),
+    );
+
+    return contentList$;
+};

client/src/components/History/providers/ContentProvider/aggregation.js

@@ -0,0 +1,158 @@
+/**
+ * Functions for aggregating incoming cache updates for presentation to the
+ * scroller. As cache changes roll in, the updates are summarized and sorted in
+ * a SkipList for fast, ordered retrieval.
+ *
+ * TODO: PouchDB has similar aggregation capabilities but when last I looked they
+ * were too difficult to get working properly with dynamically changing queries.
+ * I feel like this is the duty of the caching layer, however, and we should
+ * investigate again.
+ */
+
+import SkipList from "proper-skip-list";
+import { pipe, take, filter } from "iter-tools/es2018";
+import { SEEK } from "../../caching/enums";
+import { SearchParams } from "../../model";
+
+// utils for buildContentResult
+const distance = (a, b) => Math.abs(+a - +b);
+
+export const buildContentResult = (cfg = {}) => {
+    const { keyDirection = SEEK.ASC, pageSize = SearchParams.pageSize } = cfg;
+
+    if (!SEEK.isValid(keyDirection)) {
+        throw new Error(`buildContentResult: invalid sort direction: ${keyDirection}`);
+    }
+
+    return ([updateMap, targetKey]) => {
+        // console.log("buildContentResult", updateMap, rawTargetKey, typeof targetKey);
+
+        if (updateMap === undefined) {
+            throw new Error("missing skiplist");
+        }
+        if (targetKey === undefined) {
+            throw new Error("missing target key");
+        }
+        // make sure key is an int
+        if (!Number.isFinite(targetKey)) {
+            console.warn("Target key should be numeric", updateMap, targetKey);
+            throw new Error("buildContentResult: targetKey must be a numeric value", targetKey);
+        }
+
+        const { matchingValue, desc, asc } = updateMap.findEntries(targetKey);
+
+        // We want 3 whole pages, 1 behind us, the one we're on, and 1 ahead of us, this means
+        // we have to figure out what "behind" and "in front" mean based on the key direction
+
+        // if the keys are ascending, "down" the list is the asc iterator
+        // if the keys are ascending, "up" the list is the desc iterator
+
+        const isAscending = keyDirection == SEEK.ASC;
+        const topIterator = isAscending ? desc : asc;
+        const bottomIterator = isAscending ? asc : desc;
+
+        const topFilter = pipe(
+            filter(([k]) => (isAscending ? k < targetKey : k > targetKey)),
+            take(pageSize)
+        );
+
+        const bottomFilter = pipe(
+            filter(([k]) => (isAscending ? k > targetKey : k < targetKey)),
+            take(2 * pageSize)
+        );
+
+        const top = Array.from(topFilter(topIterator));
+        const bottom = Array.from(bottomFilter(bottomIterator));
+
+        // assemble contents array by combining the results
+        const match = matchingValue ? [[targetKey, matchingValue]] : [];
+        const contentEntries = [...top.reverse(), ...match, ...bottom];
+
+        // startKey is the closest key in the result set to the requested targetKey
+        const { startKey, startKeyIndex } = findClosestEntry(targetKey, contentEntries);
+
+        const contents = contentEntries.map(([k, v]) => v);
+
+        return {
+            // list of contents from the skiplist around the targetKey
+            contents,
+
+            // original value used to query the skiplist, should be in
+            // the middle of the contents list
+            targetKey,
+
+            // closest key to the targetKey that actually exists in the slice
+            startKey,
+
+            // index of the closest row matching the targetKey, also should
+            // be in the middle of the contents
+            startKeyIndex,
+        };
+    };
+};
+
+/**
+ * Finds closest of keys by measuring numeric distance from targetKey
+ *
+ * @param   {String|Number}  targetKey  Key to match
+ * @param   {String|Number}  entries    content entries from the map i.e. list of [key,val]
+ *
+ * @return  {Object}                    startKey, startKeyIndex
+ */
+const findClosestEntry = (targetKey, entries = []) => {
+    const startingVal = { startKey: null, startKeyIndex: -1, distance: Infinity };
+    return entries
+        .filter((entry) => Array.isArray(entry))
+        .reduce((result, entry, idx) => {
+            const [key] = entry;
+            const myDistance = distance(targetKey, key);
+            return myDistance < result.distance ? { startKey: key, startKeyIndex: idx, distance: myDistance } : result;
+        }, startingVal);
+};
+
+/**
+ * Fresh update map, gets built when inputs change
+ */
+export const newUpdateMap = () => {
+    return new SkipList();
+};
+
+/**
+ * Generates a scan function for consuming updates from the pouchdb-live-find.
+ * Adds or removes incoming docs from the skiplist
+ */
+export const processContentUpdate = (resultMap, update) => {
+    const { doc, key, match } = update;
+
+    // make sure key is an int
+    const storageKey = parseInt(key);
+
+    if (match) {
+        resultMap.upsert(storageKey, doc);
+    } else if (resultMap.has(storageKey)) {
+        resultMap.delete(storageKey);
+    }
+
+    return resultMap;
+};
+
+// Configurable function returns a key for the updateMap from a doc during upate operations
+// aggregate results in a map for each set of id + params
+export const getKeyForUpdateMap = (keyField) => (doc) => {
+    let rawKey;
+    if (keyField in doc) {
+        rawKey = doc[keyField];
+    } else {
+        // won't have a full doc on a delete, but it's embeedded in the id
+        const parts = doc._id.split("-");
+        rawKey = parseInt(parts[1]);
+    }
+
+    const result = parseInt(rawKey);
+    if (isNaN(result)) {
+        console.warn("Non integer key for document", doc);
+        throw new Error("Key for update map should be an integer");
+    }
+
+    return result;
+};

client/src/components/History/providers/ContentProvider/aggregation.test.js

@@ -1,14 +1,18 @@
 import SkipList from "proper-skip-list";
 import { buildContentResult } from "./aggregation";
+import { SEEK } from "../../caching/enums";
+
+jest.mock("app");
+jest.mock("../../caching");
 
 describe("buildContentResult", () => {
     describe("query history content from aggregate storage", () => {
         const updateList = new SkipList();
 
         // history content is keyed by hid, descending order
-        const keyDirection = "desc";
+        const keyDirection = SEEK.DESC;
         const getKey = (item) => item.hid;
-        const pageSize = 3;
+        const pageSize = 2;
 
         // sample content, indexed by hid
         const hids = [100, 95, 61, 60, 50, 44, 41, 17, 1].sort();
@@ -19,13 +23,22 @@ describe("buildContentResult", () => {
             const summarize = buildContentResult({ keyDirection, pageSize, getKey });
 
             // choose a hid we know is in the results
-            const { contents, startKey, targetKey } = summarize([updateList, queryKey]);
+            const { contents, targetKey, startKey, startKeyIndex } = summarize([updateList, queryKey]);
 
-            expect(startKey).toEqual(queryKey);
+            // returns the key we used to query the skiplist
             expect(targetKey).toEqual(queryKey);
 
+            // the closest key to targetKey in the event that targetKey does not exist
+            expect(startKey).toEqual(queryKey);
+
+            // the array index of startKey
+            expect(startKeyIndex).toEqual(2);
+
+            // pageSize causes aggregation to back up one page from start of list, then delivers
+            // the previous page, the exact match, then 2 pages after the match
             const resultKeys = contents.map(getKey);
-            expect(resultKeys).toEqual([100, 95, 61, 60, 50, 44, 41]);
+            expect(resultKeys.length).toBeLessThanOrEqual(3 * pageSize + 1);
+            expect(resultKeys).toEqual([95, 61, 60, 50, 44, 41, 17]);
         });
 
         test("query key exists, query key at top", () => {
@@ -33,13 +46,15 @@ describe("buildContentResult", () => {
             const summarize = buildContentResult({ keyDirection, pageSize, getKey });
 
             // choose a hid we know is in the results
-            const { contents, startKey, targetKey } = summarize([updateList, queryKey]);
+            const { contents, targetKey, startKey, startKeyIndex } = summarize([updateList, queryKey]);
 
             expect(startKey).toEqual(queryKey);
+            expect(startKeyIndex).toEqual(0);
             expect(targetKey).toEqual(queryKey);
 
             const resultKeys = contents.map(getKey);
-            expect(resultKeys).toEqual([100, 95, 61, 60]);
+            // match (top) + 2 pages after
+            expect(resultKeys).toEqual([100, 95, 61, 60, 50]);
         });
 
         test("query key exists, query key at bottom", () => {
@@ -47,13 +62,14 @@ describe("buildContentResult", () => {
             const summarize = buildContentResult({ keyDirection, pageSize, getKey });
 
             // choose a hid we know is in the results
-            const { contents, startKey, targetKey } = summarize([updateList, queryKey]);
+            const { contents, targetKey, startKey, startKeyIndex } = summarize([updateList, queryKey]);
 
             expect(startKey).toEqual(queryKey);
+            expect(startKeyIndex).toEqual(2);
             expect(targetKey).toEqual(queryKey);
 
             const resultKeys = contents.map(getKey);
-            expect(resultKeys).toEqual([44, 41, 17, 1]);
+            expect(resultKeys).toEqual([41, 17, 1]);
         });
 
         test("query key doesn't exist, query key in middle", () => {
@@ -61,13 +77,14 @@ describe("buildContentResult", () => {
             const summarize = buildContentResult({ keyDirection, pageSize, getKey });
 
             // choose a hid we know is in the results
-            const { contents, startKey, targetKey } = summarize([updateList, queryKey]);
+            const { contents, targetKey, startKey, startKeyIndex } = summarize([updateList, queryKey]);
 
             expect(startKey).toEqual(50);
+            expect(startKeyIndex).toEqual(2);
             expect(targetKey).toEqual(queryKey);
 
             const resultKeys = contents.map(getKey);
-            expect(resultKeys).toEqual([95, 61, 60, 50, 44, 41]);
+            expect(resultKeys).toEqual([61, 60, 50, 44, 41, 17]);
         });
 
         test("query key doesn't exist, query key near top", () => {
@@ -75,13 +92,14 @@ describe("buildContentResult", () => {
             const summarize = buildContentResult({ keyDirection, pageSize, getKey });
 
             // choose a hid we know is in the results
-            const { contents, startKey, targetKey } = summarize([updateList, queryKey]);
+            const { contents, targetKey, startKey, startKeyIndex } = summarize([updateList, queryKey]);
 
             expect(startKey).toEqual(100);
+            expect(startKeyIndex).toEqual(0);
             expect(targetKey).toEqual(queryKey);
 
             const resultKeys = contents.map(getKey);
-            expect(resultKeys).toEqual([100, 95, 61, 60]);
+            expect(resultKeys).toEqual([100, 95, 61, 60, 50]);
         });
 
         test("query key doesn't exist, query key near bottom", () => {
@@ -89,13 +107,14 @@ describe("buildContentResult", () => {
             const summarize = buildContentResult({ keyDirection, pageSize, getKey });
 
             // choose a hid we know is in the results
-            const { contents, startKey, targetKey } = summarize([updateList, queryKey]);
+            const { contents, targetKey, startKey, startKeyIndex } = summarize([updateList, queryKey]);
 
             expect(startKey).toEqual(17);
+            expect(startKeyIndex).toEqual(1);
             expect(targetKey).toEqual(queryKey);
 
             const resultKeys = contents.map(getKey);
-            expect(resultKeys).toEqual([44, 41, 17, 1]);
+            expect(resultKeys).toEqual([41, 17, 1]);
         });
 
         test("query key beyond upper limit", () => {
@@ -103,13 +122,14 @@ describe("buildContentResult", () => {
             const summarize = buildContentResult({ keyDirection, pageSize, getKey });
 
             // choose a hid we know is in the results
-            const { contents, startKey, targetKey } = summarize([updateList, queryKey]);
+            const { contents, targetKey, startKey, startKeyIndex } = summarize([updateList, queryKey]);
 
             expect(startKey).toEqual(100);
+            expect(startKeyIndex).toEqual(0);
             expect(targetKey).toEqual(queryKey);
 
             const resultKeys = contents.map(getKey);
-            expect(resultKeys).toEqual([100, 95, 61]);
+            expect(resultKeys).toEqual([100, 95, 61, 60]);
         });
 
         test("query key below lower limit", () => {
@@ -117,13 +137,14 @@ describe("buildContentResult", () => {
             const summarize = buildContentResult({ keyDirection, pageSize, getKey });
 
             // choose a hid we know is in the results
-            const { contents, startKey, targetKey } = summarize([updateList, queryKey]);
+            const { contents, targetKey, startKey, startKeyIndex } = summarize([updateList, queryKey]);
 
             expect(startKey).toEqual(1);
+            expect(startKeyIndex).toEqual(1);
             expect(targetKey).toEqual(queryKey);
 
             const resultKeys = contents.map(getKey);
-            expect(resultKeys).toEqual([41, 17, 1]);
+            expect(resultKeys).toEqual([17, 1]);
         });
     });
 
@@ -131,9 +152,9 @@ describe("buildContentResult", () => {
         const updateList = new SkipList();
 
         // collection content is keyed by element_index, ascending order
-        const keyDirection = "asc";
+        const keyDirection = SEEK.ASC;
         const getKey = (item) => item.element_index;
-        const pageSize = 3;
+        const pageSize = 2;
 
         // sample content, indexed by hid
         const element_indexes = [1, 2, 3, 4, 5, 6, 7, 8, 9];
@@ -144,13 +165,14 @@ describe("buildContentResult", () => {
             const summarize = buildContentResult({ keyDirection, pageSize, getKey });
 
             // choose a hid we know is in the results
-            const { contents, startKey, targetKey } = summarize([updateList, queryKey]);
+            const { contents, targetKey, startKey, startKeyIndex } = summarize([updateList, queryKey]);
 
             expect(startKey).toEqual(queryKey);
+            expect(startKeyIndex).toEqual(2);
             expect(targetKey).toEqual(queryKey);
 
             const resultKeys = contents.map(getKey);
-            expect(resultKeys).toEqual([2, 3, 4, 5, 6, 7, 8]);
+            expect(resultKeys).toEqual([3, 4, 5, 6, 7, 8, 9]);
         });
 
         test("query key exists, query key at top", () => {
@@ -158,13 +180,14 @@ describe("buildContentResult", () => {
             const summarize = buildContentResult({ keyDirection, pageSize, getKey });
 
             // choose a hid we know is in the results
-            const { contents, startKey, targetKey } = summarize([updateList, queryKey]);
+            const { contents, targetKey, startKey, startKeyIndex } = summarize([updateList, queryKey]);
 
             expect(startKey).toEqual(queryKey);
+            expect(startKeyIndex).toEqual(0);
             expect(targetKey).toEqual(queryKey);
 
             const resultKeys = contents.map(getKey);
-            expect(resultKeys).toEqual([1, 2, 3, 4]);
+            expect(resultKeys).toEqual([1, 2, 3, 4, 5]);
         });
 
         test("query key exists, query key at bottom", () => {
@@ -172,13 +195,14 @@ describe("buildContentResult", () => {
             const summarize = buildContentResult({ keyDirection, pageSize, getKey });
 
             // choose a hid we know is in the results
-            const { contents, startKey, targetKey } = summarize([updateList, queryKey]);
+            const { contents, targetKey, startKey, startKeyIndex } = summarize([updateList, queryKey]);
 
             expect(startKey).toEqual(queryKey);
+            expect(startKeyIndex).toEqual(2);
             expect(targetKey).toEqual(queryKey);
 
             const resultKeys = contents.map(getKey);
-            expect(resultKeys).toEqual([6, 7, 8, 9]);
+            expect(resultKeys).toEqual([7, 8, 9]);
         });
 
         test("query key doesn't exist, query key in middle", () => {
@@ -186,13 +210,14 @@ describe("buildContentResult", () => {
             const summarize = buildContentResult({ keyDirection, pageSize, getKey });
 
             // choose a hid we know is in the results
-            const { contents, startKey, targetKey } = summarize([updateList, queryKey]);
+            const { contents, targetKey, startKey, startKeyIndex } = summarize([updateList, queryKey]);
 
             expect(startKey).toEqual(4);
+            expect(startKeyIndex).toEqual(1);
             expect(targetKey).toEqual(queryKey);
 
             const resultKeys = contents.map(getKey);
-            expect(resultKeys).toEqual([2, 3, 4, 5, 6, 7]);
+            expect(resultKeys).toEqual([3, 4, 5, 6, 7, 8]);
         });
 
         test("query key doesn't exist, query key near top", () => {
@@ -200,13 +225,14 @@ describe("buildContentResult", () => {
             const summarize = buildContentResult({ keyDirection, pageSize, getKey });
 
             // choose a hid we know is in the results
-            const { contents, startKey, targetKey } = summarize([updateList, queryKey]);
+            const { contents, targetKey, startKey, startKeyIndex } = summarize([updateList, queryKey]);
 
             expect(startKey).toEqual(1);
+            expect(startKeyIndex).toEqual(0);
             expect(targetKey).toEqual(queryKey);
 
             const resultKeys = contents.map(getKey);
-            expect(resultKeys).toEqual([1, 2, 3]);
+            expect(resultKeys).toEqual([1, 2, 3, 4]);
         });
 
         test("query key doesn't exist, query key near bottom", () => {
@@ -214,13 +240,14 @@ describe("buildContentResult", () => {
             const summarize = buildContentResult({ keyDirection, pageSize, getKey });
 
             // choose a hid we know is in the results
-            const { contents, startKey, targetKey } = summarize([updateList, queryKey]);
+            const { contents, targetKey, startKey, startKeyIndex } = summarize([updateList, queryKey]);
 
             expect(startKey).toEqual(8);
+            expect(startKeyIndex).toEqual(1);
             expect(targetKey).toEqual(queryKey);
 
             const resultKeys = contents.map(getKey);
-            expect(resultKeys).toEqual([6, 7, 8, 9]);
+            expect(resultKeys).toEqual([7, 8, 9]);
         });
 
         test("query key beyond upper limit", () => {
@@ -228,13 +255,14 @@ describe("buildContentResult", () => {
             const summarize = buildContentResult({ keyDirection, pageSize, getKey });
 
             // choose a hid we know is in the results
-            const { contents, startKey, targetKey } = summarize([updateList, queryKey]);
+            const { contents, targetKey, startKey, startKeyIndex } = summarize([updateList, queryKey]);
 
             expect(startKey).toEqual(1);
+            expect(startKeyIndex).toEqual(0);
             expect(targetKey).toEqual(queryKey);
 
             const resultKeys = contents.map(getKey);
-            expect(resultKeys).toEqual([1, 2, 3]);
+            expect(resultKeys).toEqual([1, 2, 3, 4]);
         });
 
         test("query key below lower limit", () => {
@@ -242,13 +270,14 @@ describe("buildContentResult", () => {
             const summarize = buildContentResult({ keyDirection, pageSize, getKey });
 
             // choose a hid we know is in the results
-            const { contents, startKey, targetKey } = summarize([updateList, queryKey]);
+            const { contents, targetKey, startKey, startKeyIndex } = summarize([updateList, queryKey]);
 
             expect(startKey).toEqual(9);
+            expect(startKeyIndex).toEqual(1);
             expect(targetKey).toEqual(queryKey);
 
             const resultKeys = contents.map(getKey);
-            expect(resultKeys).toEqual([7, 8, 9]);
+            expect(resultKeys).toEqual([8, 9]);
         });
     });
 });

client/src/components/History/providers/ContentProvider/helpers.js

@@ -0,0 +1,56 @@
+import { SearchParams } from "../../model";
+
+// match object props
+export const propMatch = (name) => (a, b) => {
+    return a[name] === b[name];
+};
+
+// comparator for distinct() style operators inputs are an array of [id, SearchParams]
+export const inputsSame = ([a0, a1], [b0, b1]) => {
+    return a0 == b0 && SearchParams.equals(a1, b1);
+};
+
+// inputs + hid/cursor comparison
+export const loadInputsSame = ([inputsA, cursorA], [inputsB, cursorB]) => {
+    return cursorA == cursorB && inputsSame(inputsA, inputsB);
+};
+
+// dumb math util
+export const clamp = (val, [bottom, top]) => {
+    return Math.max(bottom, Math.min(top, val));
+};
+
+// simple comparators
+export const isDefined = (val) => {
+    return val !== null && val !== undefined;
+};
+
+// defined, number and finite
+export const isValidNumber = (val) => {
+    return isDefined(val) && !isNaN(val) && isFinite(val);
+};
+
+export const paginationEqual = (a, b) => {
+    return a.offset == b.offset && a.limit == b.limit;
+};
+
+/**
+ * Creates limit/offset from scroll position and data set size, adds padding. This pagination is
+ * used for the server-side content query where limit/offset will work
+ *
+ * @param   {float}  cursor         0-1 value representing how far down the list we are
+ * @param   {int}    totalMatches   total server-side known matches for content query result
+ * @param   {int}    pageSize       scaling parameter, number of rows in a "page"
+ *
+ * @return  {object}           limit/offset for use in content query
+ */
+export function buildPaginationWindow(cursor, totalMatches = 0, pageSize = SearchParams.pageSize) {
+    // this is the offset that the cursor is pointed at if we're estimating
+    const targetRow = Math.round(cursor * totalMatches);
+
+    // overlap one page before and one page after, 3 total pages queried
+    const offset = Math.max(0, targetRow - pageSize);
+    const limit = 3 * pageSize;
+
+    return { offset, limit, targetRow };
+}

client/src/components/History/providers/ContentProvider/index.js

@@ -0,0 +1,18 @@
+/**
+ * A component mixin and a set of observable utilities used in common by HicstoryContentProvider and
+ * CollectionContentProvider
+ */
+
+// base component definition for History & Collection content providers
+export { ContentProvider, defaultPayload } from "./ContentProvider";
+
+// aggreagtion functions collect an initial query and merge in subsequent updates which appear int
+// he cache over time to present a unified list of results ordered by a relevant key
+export { buildContentResult, newUpdateMap, processContentUpdate, getKeyForUpdateMap } from "./aggregation";
+export { aggregateCacheUpdates } from "./aggregateCacheUpdates";
+
+// configurable operators encapsulating shared behavior for content and collections
+export { processContentStreams } from "./processContentStreams";
+
+// utils
+export { isValidNumber, buildPaginationWindow, paginationEqual } from "./helpers";

client/src/components/History/providers/ContentProvider/processContentStreams.js

@@ -0,0 +1,61 @@
+/**
+ * General plumbing for the 2 types of content providers. This funcction emits 3 observables:
+ * loading, scrolling, and payload. Payload's the most important and the operator that genrates the
+ * payload from the inputs is passed in as a parameter to this factory.
+ */
+import { debounceTime, distinctUntilChanged, share, startWith, switchMap } from "rxjs/operators";
+import { activity, whenAny, show } from "utils/observable";
+import { propMatch } from "./helpers";
+import { SearchParams, ScrollPos } from "../../model";
+import { defaultPayload } from "../ContentProvider";
+import { BehaviorSubject } from "rxjs";
+
+/**
+ * Takes incoming history, filters and scroll position and generates loading, scrolling and payload
+ * observable for rendering the content. payload$ is an observable that emits the data that should
+ * be displaye
+ *
+ * @param   {function} payloadOperator observable operator that delivers the payload for this list
+ * @param   {object}  sources          object containing params$, history$, scrollPos$ source observables
+ * @param   {object}  settings         configuration parameter object
+ *
+ * @return  {object}                   payload$, loading$, scrolling$ observables
+ */
+// prettier-ignore
+export function processContentStreams(payloadOperator, sources = {}, settings = {}) {
+    const { debouncePeriod, debug = false } = settings;
+
+    // clean incoming source streams
+    const parent$ = sources.parent$.pipe(
+        distinctUntilChanged(propMatch("id"))
+    );
+    const params$ = sources.params$.pipe(
+        distinctUntilChanged(SearchParams.equals),
+    );
+    const pos$ = sources.scrollPos$.pipe(
+        debounceTime(debouncePeriod),
+        distinctUntilChanged(ScrollPos.equals),
+    );
+    const scrolling$ = sources.scrollPos$.pipe(
+        activity(200)
+    );
+
+    const loading$ = new BehaviorSubject(false);
+
+    // The actual loader, when history or params change we poll against the api and
+    // set up a cachewatcher, both of which are controlled by the scroll position
+    const loaderReset$ = whenAny(parent$, params$);
+    const payload$ = loaderReset$.pipe(
+        switchMap(([parent, filters]) => pos$.pipe(
+            show(debug, pos => {
+                console.clear();
+                console.log("processContentStreams: pos", JSON.stringify(pos));
+            }),
+            payloadOperator({ parent, filters, loading$, ...settings }),
+            startWith({ ...defaultPayload, placeholder: true }),
+        )),
+        share()
+    );
+
+    return { payload$, scrolling$, loading$ };
+}

client/src/components/History/providers/DscProvider/DscProvider.js

@@ -32,7 +32,8 @@ export default {
     },
     computed: {
         dsc() {
-            return new DatasetCollection(this.raw);
+            const cleanObj = JSON.parse(JSON.stringify(this.raw));
+            return new DatasetCollection(cleanObj);
         },
     },
     watch: {

client/src/components/History/providers/DscProvider/DscProvider.test.js

@@ -1,67 +1,64 @@
 /* eslint-disable no-unused-vars */
-import { wait, mountRenderless, getLocalVue } from "jest/helpers";
+import { mountRenderless, watchUntil, watchForChange } from "jest/helpers";
 import { wipeDatabase } from "../../caching";
 import { cacheContent, getCachedContent, cacheCollectionContent, getCachedCollectionContent } from "../../caching";
 import { DatasetCollection } from "../../model/DatasetCollection";
 import DscProvider from "./DscProvider";
 
 // test data as root-level collections
-import rootContents from "../../test/json/historyContentWithCollection.json";
+// import rootContents from "../../test/json/historyContentWithCollection.json";
 
 // test data for a nested collection
-import nestedContents from "../../test/json/collectionContent.json";
+// import nestedContents from "../../test/json/collectionContent.json";
+
+import { testCollection, testNestedCollection } from "../../test/testCollection";
 
 // Imports dependencies without firing up worker because that blows with Jest
 jest.mock("app");
 jest.mock("../../caching");
 
-const mountProvider = async (Component, propsData) => {
-    const localVue = getLocalVue();
-    const wrapper = mountRenderless(Component, { localVue, propsData });
-    await wrapper.vm.$nextTick();
-    return wrapper;
-};
-
 beforeEach(wipeDatabase);
 afterEach(wipeDatabase);
 
 describe("DscProvider", () => {
     let wrapper;
-    let rootCollection;
-    let nestedCollection;
-    const fakeContentUrl = "/foo/bar";
-
-    beforeEach(async () => {
-        // cache a root level collection as history content
-        const rawContent = rootContents[0];
-        const cacheResult = await cacheContent(rawContent, true);
-        rootCollection = new DatasetCollection(cacheResult);
-
-        // cache a nested collection as dscContent
-        const rawNested = nestedContents[0];
-        rawNested.parent_url = fakeContentUrl;
-        const nestedCacheResult = await cacheCollectionContent(rawNested, true);
-        nestedCollection = new DatasetCollection(nestedCacheResult);
-    });
 
     afterEach(async () => {
         if (wrapper) {
             wrapper.destroy();
         }
+        wrapper = undefined;
     });
 
     describe("monitors a root collection (direct child of a history)", () => {
+        let rootCollection;
+
         beforeEach(async () => {
-            wrapper = await mountProvider(DscProvider, {
-                collection: rootCollection,
-                isRoot: true,
+            rootCollection = await cacheContent(testCollection, true);
+            expect(rootCollection).toBeDefined();
+
+            wrapper = await mountRenderless(DscProvider, {
+                propsData: {
+                    collection: rootCollection,
+                    isRoot: true,
+                },
             });
+            expect(wrapper).toBeDefined();
+
+            const { newVal: dsc } = await watchForChange({ vm: wrapper.vm, propName: "dsc" });
+            expect(dsc._id).toEqual(rootCollection._id);
+        });
+
+        afterEach(() => {
+            rootCollection = undefined;
         });
 
         test("should successfully mount and emit the pre-cached data", () => {
-            expect(wrapper).toBeDefined();
-            const { _rev, cached_at, ...originalProps } = rootCollection;
-            const { _rev: new_rev, cached_at: new_cached_at, ...newProps } = wrapper.vm.dsc;
+            // compare output to input prop, should be the same because we didn't change anything
+            const root = new DatasetCollection(rootCollection);
+            const emitted = new DatasetCollection(wrapper.vm.dsc);
+            const { _rev, cached_at, ...originalProps } = root;
+            const { _rev: new_rev, cached_at: new_cached_at, ...newProps } = emitted;
             expect(originalProps).toEqual(newProps);
         });
 
@@ -72,10 +69,7 @@ describe("DscProvider", () => {
             const fooVal = "abc123";
             rootCollection.foo = fooVal;
             const updatedRoot = await cacheContent(rootCollection, true);
-
-            // double-check it's really changed
-            const lookup = await getCachedContent(updatedRoot._id);
-            expect(lookup.foo).toEqual(fooVal);
+            expect(updatedRoot.foo).toEqual(fooVal);
 
             // wait for next update then check the foo val
             await wrapper.vm.$nextTick();
@@ -90,16 +84,37 @@ describe("DscProvider", () => {
     });
 
     describe("monitors nested collection", () => {
+        let nestedCollection;
+        const parent_url = "/foo";
+
         beforeEach(async () => {
-            wrapper = await mountProvider(DscProvider, {
-                collection: nestedCollection,
-                isRoot: false,
+            // cache a collection root
+            nestedCollection = await cacheCollectionContent(testNestedCollection, true);
+            expect(nestedCollection).toBeDefined();
+
+            // mount and monitor
+            wrapper = await mountRenderless(DscProvider, {
+                propsData: {
+                    collection: nestedCollection,
+                    isRoot: false,
+                },
             });
+            expect(wrapper).toBeDefined();
+
+            const { newVal: dsc } = await watchForChange({ vm: wrapper.vm, propName: "dsc" });
+            expect(dsc._id).toEqual(nestedCollection._id);
+        });
+
+        afterEach(() => {
+            nestedCollection = undefined;
         });
 
         test("should successfully mount and emit the pre-cached data", () => {
-            expect(wrapper).toBeDefined();
-            expect(wrapper.vm.dsc).toEqual(nestedCollection);
+            const orig = new DatasetCollection(nestedCollection);
+            const emitted = new DatasetCollection(wrapper.vm.dsc);
+            const { _rev, cached_at, ...originalProps } = orig;
+            const { _rev: new_rev, cached_at: new_cached_at, ...newProps } = emitted;
+            expect(originalProps).toEqual(newProps);
         });
 
         test("should emit updates as they occur", async () => {
@@ -108,23 +123,15 @@ describe("DscProvider", () => {
             // write a new field to the collection
             const fooVal = "abc123";
             nestedCollection.foo = fooVal;
-            const updateResult = await cacheCollectionContent(nestedCollection);
-            expect(updateResult.updated).toBe(true);
-
-            // double-check it's really cached
-            const lookup = await getCachedCollectionContent(updateResult.id);
-            expect(lookup.foo).toEqual(fooVal);
+            const updateResult = await cacheCollectionContent(nestedCollection, true);
+            expect(updateResult.foo).toEqual(fooVal);
 
             // wait for next update then check the foo val
-            await wait(500);
+            // await wait(500);
             await wrapper.vm.$nextTick();
 
             // all same props except _rev and cached_at and foo
-            const { cached_at, _rev, foo, ...newProps } = wrapper.vm.dsc;
-            expect(foo).toEqual(fooVal);
-            expect(old_cached_at).toBeLessThan(cached_at);
-            expect(old_rev).not.toEqual(_rev);
-            expect(newProps).toEqual(oldProps);
+            expect(wrapper.vm.dsc.foo).toEqual(fooVal);
         });
     });
 });

client/src/components/History/providers/HistoryContentProvider/HistoryContentProvider.js

@@ -1,5 +1,5 @@
-import { ContentProvider } from "../ContentProvider";
-import { processHistoryStreams } from "./processHistoryStreams";
+import { ContentProvider, processContentStreams } from "../ContentProvider";
+import { contentPayload } from "./contentPayload";
 
 export default {
     mixins: [ContentProvider],
@@ -24,11 +24,11 @@ export default {
 
     methods: {
         initStreams() {
-            const { disablePoll, debouncePeriod, pageSize, params$, scrollPos$ } = this;
-            const history$ = this.watch$("history");
-            const sources = { params$, history$, scrollPos$ };
-            const settings = { disablePoll, debouncePeriod, pageSize };
-            return processHistoryStreams(sources, settings);
+            const { disablePoll, debouncePeriod, pageSize, params$, scrollPos$, debug } = this;
+            const parent$ = this.watch$("history");
+            const sources = { params$, parent$, scrollPos$ };
+            const settings = { disablePoll, debouncePeriod, pageSize, debug };
+            return processContentStreams(contentPayload, sources, settings);
         },
     },
 };

client/src/components/History/providers/HistoryContentProvider/HistoryContentProvider.test.js

@@ -1,129 +1,86 @@
-import { pipe } from "rxjs";
-import { map } from "rxjs/operators";
-import { History } from "../../model/History";
-import { SearchParams } from "../../model/SearchParams";
+import { shallowMount } from "@vue/test-utils";
+import { watchUntil, getLocalVue } from "jest/helpers";
+import { sameVueObj, payloadChange } from "../../test/providerTestHelpers";
 import HistoryContentProvider from "./HistoryContentProvider";
+import { SearchParams } from "../../model";
 import { bulkCacheContent, wipeDatabase } from "../../caching";
-import { getPropRange } from "../../caching/loadHistoryContents";
-import { watchForChange, mountRenderless } from "jest/helpers";
-import { getLocalVue } from "jest/helpers";
+import { defaultPayload } from "../ContentProvider";
+import { serverContent, testHistory, testHistoryContent } from "../../test/testHistory";
+// import { reportPayload } from "../../test/providerTestHelpers";
 
-//#region Test Data
-
-import rawHistory from "../../test/json/History.json";
-import rawHistoryContent from "../../test/json/historyContent.json";
-
-// doctor up some sample content
-const historyContent = rawHistoryContent.map((item) => {
-    item.history_id = rawHistory.id;
-    return item;
-});
-
-// set hid_counter to total length +1 because the stupid api can't count
-const testHistory = new History({
-    ...rawHistory,
-    hid_counter: historyContent[0].hid + 1,
-});
-
-//#endregion
-
-//#region Mocking
+// reads hids
+const payloadHids = (payload) => payload.contents?.map((o) => o.hid) || [];
+const localVue = getLocalVue();
 
+// mocking
 jest.mock("app");
 jest.mock("../../caching");
-
-import { loadContents } from "./loadContents";
 jest.mock("./loadContents");
 
-const loadContentsResults = historyContent;
-const { min: minHid, max: maxHid } = getPropRange(loadContentsResults, "hid");
-
-loadContents.mockImplementation((config) => {
-    return pipe(
-        map(([history, params, hid]) => {
-            return {
-                minContentHid: minHid,
-                maxContentHid: maxHid,
-                minHid: minHid,
-                maxHid: maxHid,
-                matchesUp: 0,
-                matchesDown: SearchParams.pageSize,
-                matches: SearchParams.pageSize,
-                totalMatchesUp: 0,
-                totalMatches: historyContent.length,
-            };
-        })
-    );
-});
-
-//#endregion
-
-const payloadChange = async ({ vm, label = "payload change" }) => {
-    await watchForChange({ vm, propName: "payload", label });
-    return vm.payload;
-};
-
-const mountProvider = async (Component, propsData) => {
-    const localVue = getLocalVue();
-    const wrapper = mountRenderless(Component, { localVue, propsData });
-    await wrapper.vm.$nextTick();
-    return wrapper;
-};
+afterEach(() => wipeDatabase());
 
 describe("HistoryContentProvider", () => {
     let wrapper;
 
-    const defaultProps = {
-        parent: testHistory,
-        // debouncePeriod: 0,
-        bench: 5,
-        disableLoad: true,
-    };
+    beforeEach(async () => {});
 
     afterEach(async () => {
         if (wrapper) {
             await wrapper.destroy();
         }
-        await wipeDatabase();
     });
 
-    // No loading, no watching, no polling, all output variables should be
-    // dependent solely upon input history
-
     describe("blank initialization state", () => {
         beforeEach(async () => {
-            await wipeDatabase();
-            wrapper = await mountProvider(HistoryContentProvider, defaultProps);
-        });
-
-        test("should expose all the update methods and default vars", () => {
-            // allows child components to update search params via event handler
-            expect(wrapper.vm.updateParams).toBeInstanceOf(Function);
-
-            // allows child scroller component to update the position in the history
-            // via an event handler
-            expect(wrapper.vm.setScrollPos).toBeInstanceOf(Function);
-
-            // when data comes in from the observables, these are the handlers that
-            // run in the subscription, so we can test the provider's logic without
-            // having to invoke the observables directly
-            expect(wrapper.vm.setPayload).toBeInstanceOf(Function);
-        });
-
-        test("should show an empty payload", async () => {
+            wrapper = await shallowMount(HistoryContentProvider, {
+                localVue,
+                propsData: {
+                    parent: testHistory,
+                    pageSize: 10,
+                    debug: false,
+                    debouncePeriod: 200,
+                },
+                scopedSlots: {
+                    default() {},
+                },
+            });
+            // wait for default emit
             await payloadChange({ vm: wrapper.vm });
-            expect(wrapper.vm.params).toEqual(new SearchParams());
-            expect(wrapper.vm.pageSize).toEqual(SearchParams.pageSize);
+        });
+
+        test("should expose update methods", async () => {
+            expect(wrapper.vm.setScrollPos).toBeInstanceOf(Function);
+        });
+
+        test("should show an empty payload", () => {
+            expect(SearchParams.equals(new SearchParams(), wrapper.vm.params)).toBe(true);
+            expect(sameVueObj(wrapper.vm.payload, defaultPayload));
         });
     });
 
     describe("with pre-inserted content", () => {
         beforeEach(async () => {
-            await wipeDatabase();
-            await bulkCacheContent(historyContent);
-            wrapper = await mountProvider(HistoryContentProvider, {
-                ...defaultProps,
-                pageSize: 5,
+            // eslint-disable-next-line no-unused-vars
+            const cachedContent = await bulkCacheContent(testHistoryContent, true);
+
+            wrapper = await shallowMount(HistoryContentProvider, {
+                localVue,
+                propsData: {
+                    parent: testHistory,
+                    pageSize: 10,
+                    debug: false,
+                    debouncePeriod: 200,
+                },
+                scopedSlots: {
+                    default() {},
+                },
+            });
+
+            // eslint-disable-next-line no-unused-vars
+            const spinUp = await watchUntil(wrapper.vm, function () {
+                const allDone = wrapper.vm.payload.contents.length > 0;
+                // console.log("done?", allDone);
+                return allDone;
             });
         });
 
@@ -132,16 +89,22 @@ describe("HistoryContentProvider", () => {
         // returned content rows and the total matches
 
         test("initial load, no scrolling", async () => {
-            await watchForChange({ vm: wrapper.vm, propName: "payload" });
-            const { contents, topRows, bottomRows, startKey, totalMatches } = wrapper.vm.payload;
-            // show ({ itemCount: contents.length, topRows, bottomRows, startKey, totalMatches });
+            const payload = wrapper.vm.payload;
+            // reportPayload(payload);
 
+            // test data on server
+            const allContent = serverContent();
+            // maximum hid in the fake server content
+            const maxServerHid = allContent[0].hid;
+            expect(payload).toBeDefined();
+
+            const { contents, topRows, bottomRows, startKey, totalMatches } = payload;
             expect(topRows).toEqual(0);
-            expect(contents.length).toEqual(wrapper.vm.pageSize + 1, "wrong pagesize");
+            expect(contents.length).toBeGreaterThanOrEqual(2 * wrapper.vm.pageSize);
             expect(bottomRows).toBeGreaterThan(0);
-            expect(bottomRows).toEqual(historyContent.length - contents.length);
-            expect(startKey).toEqual(maxHid);
-            expect(totalMatches).toEqual(historyContent.length);
+            expect(bottomRows).toEqual(allContent.length - contents.length);
+            expect(startKey).toEqual(maxServerHid);
+            expect(totalMatches).toEqual(allContent.length);
         });
 
         // When the scroller moves, we may end up over a known content row, in
@@ -150,11 +113,11 @@ describe("HistoryContentProvider", () => {
         // bottomRows
 
         test("scroll to known key", async () => {
-            await watchForChange({ vm: wrapper.vm, propName: "payload" });
+            const payload = wrapper.vm.payload;
 
             // shift the scroller down to half way through that content list
             // make sure it's visible and not hidden
-            const targetElement = historyContent[Math.floor(historyContent.length / 2)];
+            const targetElement = testHistoryContent[Math.floor(testHistoryContent.length / 2)];
             expect(targetElement.visible).toBe(true);
             expect(targetElement.deleted).toBe(false);
 
@@ -162,44 +125,36 @@ describe("HistoryContentProvider", () => {
             expect(targetKey).toBeDefined();
             expect(targetKey).toBeGreaterThan(0);
 
-            // set that scroll position, then look at what comes out
-            // show(targetKey);
+            // console.log("setting scroll pos to targetKey", targetKey);
             wrapper.vm.setScrollPos({ key: targetKey });
 
-            // what we should get out is a range of content around targetKey and
-            // startKey should == targetKey because we know it's in the content
-            const { contents: secondContents, ...secondPayload } = await payloadChange({
-                vm: wrapper.vm,
-                label: "second dealie",
-            });
-            // show({ items: secondContents.length, ...secondPayload });
+            // need to wait for it to settle down
+            await await payloadChange({ vm: wrapper.vm });
+
+            // check to see if what we want is there
+            const scrollPayload = wrapper.vm.payload;
+            expect(payload).toBeDefined();
+            const { startKey } = scrollPayload;
+            const hids = payloadHids(scrollPayload);
+            // console.log("hids", hids);
 
-            // should get a window around 133, 133 should be the startKey
-            const hids = secondContents.map((o) => o.hid);
             expect(hids).toContain(targetKey);
-            expect(secondPayload.startKey).toEqual(targetKey);
+            expect(startKey).toEqual(targetKey);
         });
 
         // This simulates dragging the scrollbar to a place in the history where
         // there is no exact match of HID. The provider should still render
         // contents around the targetHID, if they exist
 
-        test("should show results when we pick a HID by scroller height rather than exact match", async () => {
-            await watchForChange({ vm: wrapper.vm, propName: "payload" });
-
+        test("should show results when we pick a HID by scroller height", async () => {
             // set scroller to non-existent hid
-            wrapper.vm.setScrollPos({ cursor: 0.52323 });
-
-            const { contents, startKey, targetKey, topRows, bottomRows, totalMatches } = await payloadChange({
-                vm: wrapper.vm,
-                label: "scroll cursor dealie",
-            });
-            const hids = contents.map((o) => o.hid);
-            // show({ hids, startKey, targetKey, topRows, bottomRows, totalMatches });
+            wrapper.vm.setScrollPos({ cursor: 0.52323, key: null });
+            const scrollPayload = await payloadChange({ vm: wrapper.vm });
+            const { contents, startKey, topRows, bottomRows, totalMatches } = scrollPayload;
+            const hids = payloadHids(scrollPayload);
 
             expect(hids).toContain(startKey);
             expect(startKey).toBeGreaterThan(0);
-            expect(targetKey).toBeGreaterThan(0);
             expect(topRows).toBeGreaterThan(0);
             expect(bottomRows).toEqual(totalMatches - contents.length - topRows);
         });

client/src/components/History/providers/HistoryContentProvider/__mocks__/loadContents.js

@@ -0,0 +1,34 @@
+// common mock implementation
+// having a hard time making jest manually mock this properly
+
+import { map } from "rxjs/operators";
+import { getPropRange } from "../../../caching/loadHistoryContents";
+import { serverContent } from "../../../test/testHistory";
+
+export const loadContents = jest.fn().mockImplementation((config) => {
+    const { filters } = config;
+
+    return map((pagination) => {
+        const { offset, limit } = pagination;
+
+        // server side result set
+        const searchResults = serverContent(filters);
+        const { min: minHid, max: maxHid } = getPropRange(searchResults, "hid");
+
+        // window slice returned
+        const result = searchResults.slice(offset, offset + limit);
+        const { min: minContentHid, max: maxContentHid } = getPropRange(result, "hid");
+
+        return {
+            summary: {},
+            matches: result.length,
+            totalMatches: searchResults.length,
+            minHid,
+            maxHid,
+            minContentHid,
+            maxContentHid,
+            limit,
+            offset,
+        };
+    });
+});

client/src/components/History/providers/HistoryContentProvider/contentPayload.js

@@ -0,0 +1,258 @@
+import { merge, partition, Observable, BehaviorSubject } from "rxjs";
+import {
+    tap,
+    distinctUntilChanged,
+    map,
+    share,
+    withLatestFrom,
+    pluck,
+    publish,
+    mergeAll,
+    debounceTime,
+    window,
+} from "rxjs/operators";
+import { chunk, show } from "utils/observable";
+import { isValidNumber } from "../ContentProvider";
+import { SearchParams, CurveFit } from "../../model";
+import { loadContents } from "./loadContents";
+import { watchHistoryContents } from "./watchHistoryContents";
+
+/**
+ * Observable operator accepts history, search filters as config values, then uses an observable
+ * scroll position to periodically poll the server for new data and watch the cache for
+ * corresponding updates for a window of content defined by the cfg props.
+ *
+ * @param   {object}      cfg   history, filters, other operation parameters
+ * @return  {Observable}        Observable that emits payloads suitable for use in the scroller
+ */
+// prettier-ignore
+export const contentPayload = (cfg = {}) => {
+    const { 
+        parent: history, 
+        filters = new SearchParams(),
+        pageSize = SearchParams.pageSize, 
+        debouncePeriod = 250,
+        disablePoll = false,
+        debug = false,
+        loading$ = new BehaviorSubject(),
+    } = cfg;
+
+    // stats for this history + filters, accumulates and improves over successive polls
+    const totalMatches$ = new BehaviorSubject(0);
+    const cursorToHid$ = new BehaviorSubject(createCursorToHid(history));
+    const hidToTopRows$ = new BehaviorSubject(createHidToTopRows(history));
+
+    return publish((pos$) => {
+
+        // #region establish HID by either having exact value or estimating from scroll position
+
+        const [ noKey$, hasKey$ ] = partition(pos$, (pos) => pos.key === null);
+
+        const knownHid$ = hasKey$.pipe(
+            pluck('key')
+        );
+
+        // only have cursor height, need to make a guess
+        const estimatedHid$ = noKey$.pipe(
+            pluck('cursor'),
+            withLatestFrom(cursorToHid$),
+            map((inputs) => estimateHid(...inputs)),
+            map(hid => Math.round(hid)),
+            show(debug, hid => console.log("estimatedHid", hid)),
+        );
+
+        const hid$ = merge(knownHid$, estimatedHid$).pipe(
+            distinctUntilChanged(),
+            share(),
+        );
+        
+        //#endregion
+
+        // #region server loading
+
+        const serverHid$ = hid$.pipe(
+            chunk(pageSize, true),
+            distinctUntilChanged(),
+            show(debug, hid => console.log("serverHid", hid)),
+            share(),
+        );
+
+        const serverLoad$ = serverHid$.pipe(
+            tap(() => loading$.next(true)),
+            loadContents({ history, filters, disablePoll, debug }),
+            share(),
+        );
+
+        //#endregion
+
+        // #region Cache monitor
+
+        // after a fresh server request has been made, wait for first response before allowing
+        // an input to the cache, avoids bounciness
+        const cacheHid$ = hid$.pipe(
+            window(serverHid$),
+            mergeAll(),
+            debounceTime(100),
+        );
+        
+        const cacheMonitor$ = cacheHid$.pipe(
+            watchHistoryContents({ history, filters, pageSize, debouncePeriod, debug }),
+            show(debug, (result) => console.log("cacheMonitor.contents (hid)", result.contents.map(o => o.hid))),
+        );
+
+        const payload$ = cacheMonitor$.pipe(
+            withLatestFrom(totalMatches$, hidToTopRows$),
+            map(buildPayload(pageSize)),
+            tap(() => loading$.next(false)),
+            show(debug, (payload) => console.log("payload", payload)),
+        );
+
+        // #endregion
+
+        // #region serverLoad side-effects, keeps track of important stats via feedback loops
+
+        const newCursorToHid$ = serverLoad$.pipe(
+            withLatestFrom(cursorToHid$),
+            map(([result, fit]) => adjustCursorToHid(result, fit))
+        );
+
+        const newHidToTopRows$ = serverLoad$.pipe(
+            withLatestFrom(hidToTopRows$),
+            map(([result, fit]) => adjustHidToTopRows(result, fit))
+        );
+
+        const newMatches$ = serverLoad$.pipe(
+            pluck('totalMatches'),
+            map(val => parseInt(val)),
+        );
+
+        //#endregion
+
+        return new Observable((obs) => {
+            const sub = payload$.subscribe(obs);
+
+            // Tie stat feedback subscriptions to main payload sub
+            // Make sure not to combineLatest these subject accumulators or we'll 
+            // end up with an infinite loop
+            sub.add(newCursorToHid$.subscribe(cursorToHid$));
+            sub.add(newHidToTopRows$.subscribe(hidToTopRows$));
+            sub.add(newMatches$.subscribe(totalMatches$));
+
+            return sub;
+        })
+    })
+};
+
+const estimateHid = (cursor, fit) => {
+    // do an estimate/retrieval
+    const result = fit.get(cursor, { interpolate: true });
+    if (isValidNumber(result)) {
+        return result;
+    }
+
+    // give them the top of the data
+    if (fit.hasData) {
+        console.log("invalid curve fit", result, cursor, fit.domain);
+        return fit.get(0);
+    }
+
+    // we're screwed
+    console.log("estimateHid no estimate available");
+    return undefined;
+};
+
+const adjustCursorToHid = (result, fit) => {
+    const { maxHid, minHid, maxContentHid, minContentHid, matches, totalMatchesUp, totalMatches } = result;
+
+    // set 0, 1 for min/max hids
+    fit.set(0, maxHid);
+    fit.set(1, minHid);
+
+    // find cursor for top & bottom of returned content
+    if (totalMatches > 0) {
+        fit.set(totalMatchesUp / totalMatches, maxContentHid);
+        fit.set((matches + totalMatchesUp) / totalMatches, minContentHid);
+    }
+
+    return fit;
+};
+
+const adjustHidToTopRows = (result, fit) => {
+    const {
+        // min and max of returned data
+        minContentHid,
+        maxContentHid,
+
+        // endpoints for all of history, that match filters
+        minHid,
+        maxHid,
+
+        // num matches up/down from request hid
+        matchesUp,
+        matchesDown,
+        matches,
+
+        // total matches up/down from request hid
+        totalMatchesUp,
+        totalMatches,
+    } = result;
+
+    if (matches > 0) {
+        fit.set(maxContentHid, totalMatchesUp - matchesUp);
+        fit.set(minContentHid, Math.max(0, totalMatchesUp + matchesDown - 1));
+    }
+
+    fit.set(maxHid, 0);
+    if (totalMatches > 0) {
+        fit.set(minHid, totalMatches - 1);
+    }
+
+    fit.domain = [minHid, maxHid];
+
+    return fit;
+};
+
+const buildPayload = (pageSize) => (inputs) => {
+    // cacheWatchResult is the value from the cache watcher
+    // hidToTopRows is a dataset for estimating toprows from HID
+    // since we can't possibly have that information at the time
+    // of the cache-read
+    const [cacheWatchResult, totalMatches, hidToTopRows] = inputs;
+
+    // contents is the list from the aggregator
+    // targetKey was the request value for the aggregation
+    const { contents = [] } = cacheWatchResult;
+
+    let topRows = 0;
+    let bottomRows = Math.max(0, totalMatches - contents.length - 1);
+
+    // missing rows above & below content
+    if (hidToTopRows.size && contents.length && totalMatches > pageSize) {
+        const maxHid = contents[0].hid;
+        const rowsAboveTop = hidToTopRows.get(maxHid, { interpolate: true });
+        topRows = Math.max(0, Math.round(rowsAboveTop));
+        bottomRows = Math.max(0, totalMatches - topRows - contents.length);
+    }
+
+    return { topRows, bottomRows, totalMatches, ...cacheWatchResult };
+};
+
+const createHidToTopRows = (history) => {
+    const fit = new CurveFit();
+    fit.domain = [0.0, +history.hidItems];
+    fit.xPrecision = 3;
+    fit.yPrecision = 3;
+    fit.set(history.hidItems, 0.0);
+    fit.set(0.0, history.hidItems);
+    return fit;
+};
+
+const createCursorToHid = (history) => {
+    const fit = new CurveFit();
+    fit.domain = [0.0, 1.0];
+    fit.xPrecision = 3;
+    fit.yPrecision = 2;
+    fit.set(0.0, history.hidItems);
+    fit.set(1.0, 1);
+    return fit;
+};

client/src/components/History/providers/HistoryContentProvider/contentPayload.test.js

@@ -0,0 +1,153 @@
+import { BehaviorSubject, timer } from "rxjs";
+import { takeUntil, share } from "rxjs/operators";
+import { ObserverSpy } from "@hirez_io/observer-spy";
+import { ScrollPos } from "../../model";
+import { bulkCacheContent, wipeDatabase } from "../../caching";
+import { contentPayload } from "./contentPayload";
+import { loadContents } from "./loadContents";
+import { serverContent, testHistory, testHistoryContent } from "../../test/testHistory";
+import { untilNthEmission } from "jest/helpers";
+// import { reportPayload } from "../../test/providerTestHelpers";
+
+jest.mock("app");
+jest.mock("../../caching");
+jest.mock("./loadContents");
+
+// Create a history and a set of filters then wire up a scrollPos to
+// the contentPayload operator, check the payloads that come out
+
+describe("contentPayload operator", () => {
+    let sub;
+    const safetyTimeout = 2000;
+    const disablePoll = true;
+    const debouncePeriod = 250;
+
+    beforeEach(async () => {
+        await wipeDatabase();
+        await bulkCacheContent(testHistoryContent, true);
+    });
+
+    afterEach(async () => {
+        if (sub) sub.unsubscribe();
+        await wipeDatabase();
+    });
+
+    // The first emission any time we subscribe to the payload should
+    // be the top of the history, this is because we do not yet have any
+    // way to gauge how big the history is, so our first emission is always
+    // from the top of the list, which is where we reset the scroller when
+    // the inputs change anyway
+
+    test("single emission, no scrolling, top of list", async () => {
+        const spy = new ObserverSpy();
+        const start = new ScrollPos();
+        const input$ = new BehaviorSubject(start);
+        const pageSize = 10;
+
+        // prettier-ignore
+        sub = input$.pipe(
+            contentPayload({ 
+                parent: testHistory, 
+                pageSize,
+                disablePoll,
+                debouncePeriod
+            }), 
+            takeUntil(timer(safetyTimeout))
+        ).subscribe(spy);
+
+        await spy.onComplete();
+
+        // should finish
+        expect(spy.receivedComplete()).toBe(true);
+
+        // should call the loader once
+        expect(loadContents.mock.calls.length).toEqual(1);
+
+        // should only emit one result
+        expect(spy.receivedNext()).toBe(true);
+        expect(spy.getValues().length).toEqual(1);
+
+        // expect a payload with results from the top of the list (cursor = 0)
+        const { contents } = spy.getFirstValue();
+
+        expect(contents).toBeDefined();
+        expect(contents).toBeInstanceOf(Array);
+        expect(contents.length).toBeGreaterThanOrEqual(2 * pageSize);
+
+        const payloadHids = contents.map((c) => c.hid);
+        expect(payloadHids[0]).toEqual(testHistoryContent[0].hid);
+    });
+
+    // The first emission must be from the top of the list, but then
+    // the user moves the scroller down half-way. Should get results
+    // from the middle of the sample contents
+
+    test("move half way down the list", async () => {
+        const pageSize = 5;
+
+        // scroll position (cursor/key)
+        const pos$ = new BehaviorSubject(ScrollPos.create());
+
+        // subscribe to payload emitter
+        // first emission needs to be at top of list
+        const payload$ = pos$.pipe(
+            contentPayload({
+                parent: testHistory,
+                pageSize,
+                disablePoll,
+            }),
+            takeUntil(timer(safetyTimeout)),
+            share()
+        );
+
+        const spy = new ObserverSpy();
+        sub = payload$.subscribe(spy);
+
+        // wait until first emission, then move scroller
+        await untilNthEmission(payload$, 1);
+        pos$.next(ScrollPos.create({ cursor: 0.5 }));
+
+        await spy.onComplete();
+        expect(spy.receivedComplete()).toBe(true);
+        expect(spy.receivedNext()).toBe(true);
+        // spy.getValues().forEach(reportPayload);
+
+        // should call the loader once
+        expect(loadContents.mock.calls.length).toBeGreaterThan(0);
+
+        const serverResults = serverContent();
+        const serverResultHids = serverResults.map((o) => o.hid);
+
+        const initialPayload = spy.getFirstValue();
+        {
+            const { contents, startKey, startKeyIndex, topRows, bottomRows, totalMatches } = initialPayload;
+            const listLength = 2 * pageSize;
+            const payloadHids = contents.map((o) => o.hid);
+            const historyHids = serverResultHids.slice(startKeyIndex, contents.length);
+
+            expect(contents).toBeDefined();
+            expect(contents).toBeInstanceOf(Array);
+            expect(contents.length).toBeGreaterThanOrEqual(listLength);
+            expect(startKey).toEqual(serverResultHids[0]);
+            expect(startKeyIndex).toEqual(0);
+            expect(topRows).toEqual(0);
+            expect(totalMatches).toEqual(serverResults.length);
+            expect(payloadHids).toEqual(historyHids);
+            expect(topRows + bottomRows + contents.length).toEqual(totalMatches);
+        }
+
+        const secondPayload = spy.getLastValue();
+        {
+            const { contents, startKey, startKeyIndex, topRows, bottomRows, totalMatches } = secondPayload;
+            const payloadHids = contents.map((o) => o.hid);
+
+            expect(contents).toBeDefined();
+            expect(contents).toBeInstanceOf(Array);
+            expect(topRows).toBeGreaterThan(0);
+            expect(totalMatches).toEqual(serverResults.length);
+            expect(startKeyIndex).toBeGreaterThan(0);
+            expect(payloadHids).toContain(startKey);
+            expect(topRows + bottomRows + contents.length).toEqual(totalMatches);
+        }
+    });
+});

client/src/components/History/providers/HistoryContentProvider/loadContents.js

@@ -1,22 +1,37 @@
-import { defer, of } from "rxjs";
-import { repeat, switchMap, switchMapTo, startWith, debounceTime } from "rxjs/operators";
-import { decay } from "utils/observable/decay";
+import { concat, defer, of, throwError } from "rxjs";
+import { delay, repeat, switchMap, switchMapTo, debounceTime, startWith } from "rxjs/operators";
+import { decay } from "utils/observable";
 import { monitorXHR } from "utils/observable/monitorXHR";
 import { loadHistoryContents } from "../../caching";
+import { SearchParams } from "../../model";
 
 // prettier-ignore
 export const loadContents = (cfg = {}) => {
     const {
-        windowSize,
+        history,
+        filters,
         initialInterval = 2 * 1000,
         maxInterval = 10 * initialInterval,
-        disablePoll = false
+        disablePoll = false,
+        // debug = false,
+        // time between initial load and polling start
+        startingDelay = 5000,
+        windowSize = 2 * SearchParams.pageSize,
     } = cfg;
 
-    return switchMap(([{ id }, params, hid]) => {
+    if (history === undefined) {
+        return throwError(new Error("Missing history in loadContents"));
+    }
+    if (filters === undefined) {
+        return throwError(new Error("Missing filters in loadContents"));
+    }
+
+    const { id } = history;
+
+    return switchMap((hid) => {
 
         // a single history update
-        const singleLoad$ = of([id, params, hid]).pipe(
+        const singleLoad$ = of([id, filters, hid]).pipe(
             loadHistoryContents({ windowSize })
         );
 
@@ -39,6 +54,13 @@ export const loadContents = (cfg = {}) => {
             switchMapTo(freshPoll$)
         );
 
-        return disablePoll ? singleLoad$ : poll$;
-    });
+        // if we're polling, add a little delay between initial load and polling
+        // avoids pointless polling if the user is rapidly changing the view
+        const polling$ = of(true).pipe(
+            delay(startingDelay),
+            switchMapTo(poll$)
+        );
+
+        return disablePoll ? singleLoad$ : concat(singleLoad$, polling$);
+    })
 };

client/src/components/History/providers/HistoryContentProvider/processHistoryStreams.js

@@ -1,224 +0,0 @@
-import { combineLatest, merge, Subject } from "rxjs";
-import { tap, pluck, map, distinctUntilChanged, withLatestFrom, debounceTime, share } from "rxjs/operators";
-import { tag } from "rxjs-spy/operators/tag";
-import { whenAny } from "utils/observable/whenAny";
-import { activity } from "utils/observable/activity";
-import { shareButDie } from "utils/observable/shareButDie";
-import { chunk } from "../../caching/operators/chunk";
-import { nth } from "utils/observable/nth";
-
-import { SearchParams, CurveFit } from "../../model";
-import { watchHistoryContents } from "./watchHistoryContents";
-import { loadContents } from "./loadContents";
-import { isValidNumber, inputsSame } from "../ContentProvider";
-
-// inputs + hid/cursor comparison
-export const loadInputsSame = ([inputsA, cursorA], [inputsB, cursorB]) => {
-    return cursorA == cursorB && inputsSame(inputsA, inputsB);
-};
-
-// prettier-ignore
-export function processHistoryStreams(sources, settings) {
-    const { disablePoll, debouncePeriod, pageSize } = settings;
-    const { params$: rawParams$, history$: rawHistory$, scrollPos$ } = sources;
-
-    //#region Clean up, filter, debounce raw inputs
-
-    const history$ = rawHistory$.pipe(distinctUntilChanged((a, b) => a.id == b.id));
-    const params$ = rawParams$.pipe(distinctUntilChanged(SearchParams.equals));
-    const inputs$ = whenAny(history$, params$).pipe(share());
-    const scrolling$ = scrollPos$.pipe(activity());
-
-    //#endregion
-
-    //#region Total Matches, from history and returned ajax stats
-
-    const serverTotalMatches$ = new Subject();
-    const historyMatches$ = history$.pipe(pluck("hidItems"));
-    const totalMatches$ = merge(historyMatches$, serverTotalMatches$).pipe(distinctUntilChanged());
-
-    //#endregion
-
-    //#region Curve-fit objects, used to estimate hid & topRows
-
-    const updateCursorToHid$ = new Subject();
-    const updateHidToTopRows$ = new Subject();
-
-    // reset curve-fit data whenever inputs change
-    // same as history but emits each time inputs emits
-    const h$ = inputs$.pipe(nth(0));
-    const freshCursorToHid$ = h$.pipe(map(createCursorToHid));
-    const freshHidToTopRows$ = h$.pipe(map(createHidToTopRows));
-
-    // merge create + manual updates
-    const cursorToHid$ = merge(freshCursorToHid$, updateCursorToHid$).pipe(shareButDie(1));
-    const hidToTopRows$ = merge(freshHidToTopRows$, updateHidToTopRows$).pipe(shareButDie(1));
-
-    //#endregion
-
-    //#region Estimate HID from cursor + history + curveFit
-
-    const hid$ = whenAny(scrollPos$, cursorToHid$).pipe(
-        map(([pos, fit]) => (pos.key ? pos.key : estimateHid(pos, fit))),
-        map((hid) => parseInt(hid)),
-        distinctUntilChanged(),
-        tag('hid'),
-        shareButDie(1)
-    );
-
-    //#endregion
-
-    //#region Loading
-
-    const chunkedHid$ = hid$.pipe(chunk(pageSize, true));
-
-    const loadInput$ = whenAny(inputs$, chunkedHid$).pipe(
-        distinctUntilChanged(loadInputsSame),
-        map(([inputs, hid]) => [...inputs, hid]),
-        shareButDie(1)
-    );
-
-    const loadResult$ = loadInput$.pipe(
-        loadContents({ disablePoll, windowSize: 2 * pageSize })
-    );
-
-    // side-effects of loading
-    const loadEffects$ = loadResult$.pipe(
-        withLatestFrom(cursorToHid$, hidToTopRows$),
-        tap(([result, cursorToHid, hidToTopRows]) => {
-            // can result in infinite loops, adjust curve-data by reference
-            adjustCursorToHid(result, cursorToHid);
-            adjustHidToTopRows(result, hidToTopRows);
-            // updateHidToTopRows$.next();
-            serverTotalMatches$.next(result.totalMatches);
-        })
-    );
-
-    // loading flag
-    const loading$ = merge(loadInput$, loadEffects$).pipe(activity());
-
-    //#endregion
-
-    //#region Use inputs to read from cache
-
-    const cacheFromMonitor$ = inputs$.pipe(
-        watchHistoryContents({
-            hid$,
-            pageSize,
-            debouncePeriod,
-        })
-    );
-
-    const payload$ = combineLatest(cacheFromMonitor$, hidToTopRows$, totalMatches$).pipe(
-        debounceTime(debouncePeriod),
-        map((inputs) => buildPayload(...inputs, pageSize))
-    );
-
-    //#endregion
-
-    return { payload$, scrolling$, loading$ };
-}
-
-function estimateHid(scrollPos, cursorToHid) {
-    const { cursor, key } = scrollPos;
-    const fit = cursorToHid;
-
-    // do an estimate/retrieval
-    const result = fit.get(cursor, { interpolate: true });
-    if (isValidNumber(result)) {
-        return result;
-    }
-
-    // give them the top of the data
-    if (fit.hasData) {
-        console.log("invalid curve fit", result, cursor, key, fit.domain);
-        return fit.get(0);
-    }
-
-    // we're screwed
-    return undefined;
-}
-
-function adjustCursorToHid(result, fit) {
-    const { maxHid, minHid, maxContentHid, minContentHid, matches, totalMatchesUp, totalMatches } = result;
-
-    // set 0, 1 for min/max hids
-    fit.set(0, maxHid);
-    fit.set(1, minHid);
-
-    // find cursor for top & bottom of returned content
-    if (totalMatches > 0) {
-        fit.set(totalMatchesUp / totalMatches, maxContentHid);
-        fit.set((matches + totalMatchesUp) / totalMatches, minContentHid);
-    }
-}
-
-function adjustHidToTopRows(result, fit) {
-    const {
-        // min and max of returned data
-        minContentHid,
-        maxContentHid,
-
-        // endpoints for all of history, that match filters
-        minHid,
-        maxHid,
-
-        // num matches up/down from request hid
-        matchesUp,
-        matchesDown,
-        matches,
-
-        // total matches up/down from request hid
-        totalMatchesUp,
-        totalMatches,
-    } = result;
-
-    if (matches > 0) {
-        fit.set(maxContentHid, totalMatchesUp - matchesUp);
-        fit.set(minContentHid, Math.max(0, totalMatchesUp + matchesDown - 1));
-    }
-
-    fit.set(maxHid, 0);
-    if (totalMatches > 0) {
-        fit.set(minHid, totalMatches - 1);
-    }
-
-    fit.domain = [minHid, maxHid];
-}
-
-function buildPayload(result, hidToTopRows, totalMatches, pageSize) {
-    const { contents = [], startKey = null, targetKey } = result;
-    const fit = hidToTopRows;
-
-    let topRows = 0;
-    let bottomRows = Math.max(0, totalMatches - contents.length);
-    // missing rows above & below content
-    if (fit.size && contents.length && totalMatches > pageSize) {
-        const maxHid = contents[0].hid;
-        const rowsAboveTop = fit.get(maxHid, { interpolate: true });
-        topRows = Math.max(0, Math.round(rowsAboveTop));
-        bottomRows = Math.max(0, totalMatches - topRows - contents.length);
-    }
-
-    return { contents, startKey, targetKey, topRows, bottomRows, totalMatches };
-}
-
-function createHidToTopRows(history) {
-    const fit = new CurveFit();
-    fit.domain = [0.0, +history.hidItems];
-    fit.xPrecision = 3;
-    fit.yPrecision = 3;
-    fit.set(history.hidItems, 0.0);
-    fit.set(0.0, history.hidItems);
-    return fit;
-}
-
-function createCursorToHid(history) {
-    const fit = new CurveFit();
-    fit.domain = [0.0, 1.0];
-    fit.xPrecision = 3;
-    fit.yPrecision = 2;
-    fit.set(0.0, history.hidItems);
-    fit.set(1.0, 1);
-    return fit;
-}

client/src/components/History/providers/HistoryContentProvider/readme.md

@@ -0,0 +1,73 @@
+The history content provider is where the grunt-work of loading, polling and caching
+happens. The output of the provider is a list of current data suitable for rendering in the companion Scroller component.
+
+### Design
+
+The API of the provider is a renderless Vue component. Its inputs are the history and the
+currently selected filters. Internally, it tracks the user's current scroll position on the scrolling list then combines those 3 values to ultimately expose an object containing all the
+currently-visible content for the history along with some other values that are useful
+for the scroller to render the visible window of data.
+
+#### Composition
+```html static
+<HistoryContentProvider
+    :parent="history" 
+    :params="params" 
+    v-slot="{ setScrollPos, loading, scrolling, payload }">
+
+    <!-- other UI elements that care about the current content,
+    filters editors, etc -->
+
+    <Scroller 
+        :items="payload.contents"
+        :top-placeholders="payload.topRows"
+        :bottom-placeholders="payload.bottomRows"
+        :scroll-to="payload.startKey"
+        @scroll="setScrollPos">
+
+        <template v-slot="{ item, index }">
+            <!-- single item rendering -->
+        </template>
+
+    </Scroller>
+
+</HistoryContentProvider>
+```
+
+#### History Content Provider exposed slot props (outputs)
+```js
+{
+    // a function to update the internal scroll position
+    setScrollPos: fn,
+
+    // current content window
+    payload: {
+        contents: [] 
+        topRows: 10 
+        botttomRows: 100,
+        startKey: "123",
+    },
+
+    // flags
+    loading: true,
+    scrolling: false,
+}
+```
+
+### Implementation
+
+Because actually delivering that data is kind of a complex process (see dependency graph below) and has multiple goals, Vue's rather
+simplistic observability system is not by itself be an ideal means to actually deliver the
+constantly shifting payload value. Instead, we used RxJS and its more sophisticated, but admittedly
+more complex API.
+
+The goal of this section is to explain how the RxJS works inside the history content provider to
+deliver the payload from the 3 changing inputs.
+
+I believe it's easiest to follow RxJS streams by simply drawing a map of them. An rxjs operator is a
+functional transformation of a source observable emitter into a new observable emitter and therefore
+a combined composition of them (often called a stream) is essentially its own dependency tree. All
+you have to do is connect the inputs to the outputs.
+
+### RxJS Flowchart
+

client/src/components/History/providers/HistoryContentProvider/watchHistoryContents.js

@@ -1,77 +1,37 @@
-import { of, combineLatest } from "rxjs";
-import { map, switchMap, debounceTime, scan, distinctUntilChanged } from "rxjs/operators";
-import { chunk } from "../../caching/operators/chunk";
+import { of } from "rxjs";
+import { map } from "rxjs/operators";
+import { aggregateCacheUpdates } from "../ContentProvider";
 import { monitorHistoryContent } from "../../caching";
-import { SearchParams } from "../../model/SearchParams";
-import { processContentUpdate, newUpdateMap, buildContentResult, getKeyForUpdateMap } from "../aggregation";
+import { SEEK } from "../../caching/enums";
+import { SearchParams } from "../../model";
 
-// disable debugging
-const tag = () => (src$) => src$;
-
-/**
- * Monitor history in the region of the cursor for the provided inputs
- *
- * @param {Observable} hid$ HID observable, HID of first row of scroller
- * @param {Object} cfg Config object, see below
- */
 // prettier-ignore
-export const watchHistoryContents = (cfg = {}) => input$ => {
-    const {
-        hid$,
-
-        // final page size for rendered content
-        pageSize = SearchParams.pageSize,
-
-        // input and output throttling
-        debouncePeriod = 250,
-        outputDebounce = debouncePeriod,
-
-        // field and key order for the contents
+export const watchHistoryContents = (cfg = {}) => hid$ => {
+    const { 
+        history, 
+        filters = new SearchParams(),
+        pageSize = SearchParams.pageSize, 
+        debug = false,
         keyField = "hid",
-        keyDirection = "desc",
-
-        // input hid chunk, reduces the number of monitors we create
-        // width of the hid window to observe in the cache
-        monitorChunk = 5 * pageSize,
-        monitorPageSize = 5 * monitorChunk,
+        keyDirection = SEEK.DESC,
+        ...otherConfig 
     } = cfg;
+    
+    // builds a monitor observable based on a hid
+    const monitor = (hid) => of([history.id, filters, hid]).pipe(
+        monitorHistoryContent({ pageSize, debug, keyField })
+    );
 
-    const getKey = getKeyForUpdateMap(keyField);
-    const aggregator = processContentUpdate({ getKey });
-    const summarize = buildContentResult({ pageSize, keyDirection, getKey });
-
-    const contentMap$ = input$.pipe(
-        switchMap(([{id}, params]) => {
-
-            // extremely wide chunking for the monitor since that's local
-            // and assembling a new monitor is expensive. New monitors will be
-            // created each time hid$ emits
-            const monitorInput$ = hid$.pipe(
-                chunk(monitorChunk, true),
-                distinctUntilChanged(),
-                map(hid => [id, params, hid]),
-                tag('watchHistoryContents-monitorInputs'),
-            );
-
-            const monitorOutput$ = monitorInput$.pipe(
-                switchMap(monitorInput => of(monitorInput).pipe(
-                    monitorHistoryContent({
-                        pageSize: monitorPageSize
-                    }),
-                )),
-                scan(aggregator, newUpdateMap()),
-            );
-
-            return monitorOutput$;
+    // handles the plumbing for observing the query (monitor) and collecting the output into an
+    // aggregate ordered list focused around the current target key (hid in this case)
+    return hid$.pipe(
+        map(val => parseInt(val)),
+        aggregateCacheUpdates(monitor,  {
+            keyField,
+            keyDirection,
+            pageSize, 
+            debug,
+            ...otherConfig 
         }),
     );
-
-    // take a slice out of that content map corresponding to the current hid
-    const contentWindow$ = combineLatest([contentMap$, hid$]).pipe(
-        debounceTime(outputDebounce),
-        map(summarize),
-        tag('watchHistoryContents-contentWindow'),
-    );
-
-    return contentWindow$;
 };

client/src/components/History/providers/HistoryContentProvider/watchHistoryContents.test.js

@@ -1,7 +1,7 @@
 import { of, isObservable, timer } from "rxjs";
 import { take, takeUntil } from "rxjs/operators";
 import { ObserverSpy } from "@hirez_io/observer-spy";
-// import { wait } from "jest/helpers";
+import { untilNthEmission } from "jest/helpers";
 
 import { SearchParams } from "../../model/SearchParams";
 import { buildContentId } from "../../caching/db/observables";
@@ -17,31 +17,39 @@ import historyContent from "../../test/json/historyContent.json";
 jest.mock("app");
 jest.mock("../../caching");
 
-beforeEach(wipeDatabase);
-afterEach(wipeDatabase);
+beforeEach(async () => {
+    await wipeDatabase();
+    // eslint-disable-next-line no-unused-vars
+    const cachedContent = await bulkCacheContent(historyContent, true);
+    // console.log(cachedContent.map((o) => o.hid));
+});
+
+afterEach(async () => {
+    await wipeDatabase();
+});
 
 // We're testing observables and we're assuming they complete,
 // but we add a takeUntil() to each one in the event that something
 // is wrong to avoid making the tests take forever
-const safetyTimeout = 600;
+const safetyTimeout = 1000;
 
 describe("watchHistoryContents", () => {
     const firstDoc = historyContent[0];
 
-    beforeEach(async () => await bulkCacheContent(historyContent));
-
     describe("simple loading scenario", () => {
         const hid$ = of(firstDoc.hid);
-        const fakeHistory = { id: firstDoc.history_id };
-        const input$ = of([fakeHistory, new SearchParams()]);
+        const history = { id: firstDoc.history_id };
+        const filters = new SearchParams();
+        const pageSize = 5;
 
         test("should observe pre-inserted content on the initial emission", async () => {
-            const watcher$ = input$.pipe(
+            const watcher$ = hid$.pipe(
                 watchHistoryContents({
-                    hid$,
-                    pageSize: 5,
+                    history,
+                    filters,
+                    pageSize,
+                    debug: false,
                 }),
-                take(1),
                 takeUntil(timer(safetyTimeout))
             );
 
@@ -54,54 +62,54 @@ describe("watchHistoryContents", () => {
 
             expect(spy.receivedNext()).toBe(true);
             expect(spy.receivedComplete()).toBe(true);
+            expect(spy.getValuesLength()).toBe(1);
 
             const { startKey, contents } = spy.getFirstValue();
-            expect(startKey).toEqual(firstDoc.hid); // we know there should be an exactd match
+            // we know there should be an exactd match
+            expect(startKey).toEqual(firstDoc.hid);
             expect(Array.isArray(contents)).toEqual(true);
-            expect(contents.length).toEqual(6); // we're at the top, should get 5 rows down + the match
+            // Top of list, match + 2 * pageSize
+            expect(contents.length).toBeGreaterThanOrEqual(2 * pageSize);
         });
 
         test("should observe subsequently updated data", async () => {
-            const watcher$ = input$.pipe(
+            const watcher$ = hid$.pipe(
                 watchHistoryContents({
-                    hid$,
+                    history,
+                    filters,
+                    debouncePeriod: 100,
                     pageSize: 5,
                 }),
-                // safeguard, sometimes the debouncing merges both results into one
-                // event, sometimes they come in as 2 separate ones, either way
-                // should be done in 2 secs.
-                take(2),
-                takeUntil(timer(1000))
+                takeUntil(timer(safetyTimeout))
             );
 
-            expect(isObservable(watcher$)).toBe(true);
-
-            // after the initial event, update one of the items
-
-            const firstWatcherEvent$ = watcher$.pipe(take(1));
-
-            firstWatcherEvent$.subscribe(async () => {
-                const docId = buildContentId(firstDoc);
-                const lookup = await getCachedContent(docId);
-                expect(lookup.hid).toEqual(firstDoc.hid);
-                expect(lookup._id).toEqual(docId);
-
-                lookup.foo = "abc";
-                const update = await cacheContent(lookup);
-                expect(update.updated).toEqual(true);
-                expect(update.id).toEqual(lookup._id);
-                expect(update.id).toEqual(docId);
-            });
-
             // spy on output of observable, wait for it to end
             const spy = new ObserverSpy();
             watcher$.subscribe(spy);
-            await spy.onComplete();
+
+            // wait for the first event (take(n) as a promise for easier testing)
+            await untilNthEmission(watcher$, 1, safetyTimeout);
+
+            // then update the first doc, need to delay longer than
+            // the debounce on the watch operator or the aggregation will
+            // think it's all part of the same update
+            const docId = buildContentId(firstDoc);
+            const lookup = await getCachedContent(docId);
+            expect(lookup.hid).toEqual(firstDoc.hid);
+            expect(lookup._id).toEqual(docId);
+
+            lookup.foo = "abc";
+            const update = await cacheContent(lookup);
+            expect(update.updated).toEqual(true);
+            expect(update.id).toEqual(lookup._id);
+            expect(update.id).toEqual(docId);
 
             // Should see 2 sets of content, the last of which should have the
             // foo field in one of its entries
+            await spy.onComplete();
             expect(spy.receivedNext()).toBe(true);
             expect(spy.receivedComplete()).toBe(true);
+            expect(spy.getValuesLength()).toBe(2);
 
             const { contents: lastContents } = spy.getLastValue();
             expect(lastContents.some((doc) => doc.foo == "abc")).toBe(true);
@@ -110,17 +118,20 @@ describe("watchHistoryContents", () => {
 
     describe("should respect filter inputs", () => {
         const hid$ = of(firstDoc.hid);
-        const fakeHistory = { id: firstDoc.history_id };
+        const history = { id: firstDoc.history_id };
+        let filters;
+
+        beforeEach(() => {
+            filters = new SearchParams();
+        });
 
         test("deleted flag", async () => {
-            const filters = new SearchParams();
             filters.showDeleted = true;
 
-            const input$ = of([fakeHistory, filters]);
-
-            const watcher$ = input$.pipe(
+            const watcher$ = hid$.pipe(
                 watchHistoryContents({
-                    hid$,
+                    history,
+                    filters,
                     pageSize: 5,
                 }),
                 take(1),
@@ -142,14 +153,12 @@ describe("watchHistoryContents", () => {
         });
 
         test("hidden flag", async () => {
-            const filters = new SearchParams();
             filters.showHidden = true;
 
-            const input$ = of([fakeHistory, filters]);
-
-            const watcher$ = input$.pipe(
+            const watcher$ = hid$.pipe(
                 watchHistoryContents({
-                    hid$,
+                    history,
+                    filters,
                     pageSize: 5,
                 }),
                 take(1),
@@ -171,15 +180,13 @@ describe("watchHistoryContents", () => {
         });
 
         test("deleted and hidden flag", async () => {
-            const filters = new SearchParams();
             filters.showDeleted = true;
             filters.showHidden = true;
 
-            const input$ = of([fakeHistory, filters]);
-
-            const watcher$ = input$.pipe(
+            const watcher$ = hid$.pipe(
                 watchHistoryContents({
-                    hid$,
+                    history,
+                    filters,
                     pageSize: 5,
                 }),
                 take(1),

client/src/components/History/providers/aggregation.js

@@ -1,158 +0,0 @@
-/**
- * Functions for aggregating incoming cache updates for presentation to the
- * scroller. As cache changes roll in, the updates are summarized and sorted in
- * a SkipList for fast retrieval.
- */
-
-import SkipList from "proper-skip-list";
-import { pipe, take, filter } from "iter-tools/es2018";
-import { ACTIONS } from "../caching/db/monitorQuery";
-
-// utils for buildContentResult
-const firstEntryKey = (entries) => (entries.length ? entries[0][0] : undefined);
-const distance = (a, b) => Math.abs(+a - +b);
-
-export const buildContentResult = (cfg = {}) => {
-    const { keyDirection = "asc", pageSize = 50, getKey } = cfg;
-
-    if (keyDirection != "asc" && keyDirection != "desc") {
-        throw new Error(`buildContentResult: invalid sort direction: ${keyDirection}`);
-    }
-
-    return ([updateMap, targetKey]) => {
-        if (isNaN(targetKey)) {
-            console.warn("Target key should be numeric", updateMap, targetKey);
-            throw new Error("buildContentResult: targetKey must be a numeric value", targetKey);
-        }
-
-        const { matchingValue, desc, asc } = updateMap.findEntries(targetKey);
-
-        // iterator transformations to find some up/down from target
-        const up = pipe(
-            filter(([k]) => k > targetKey),
-            take(pageSize)
-        );
-        const down = pipe(
-            filter(([k]) => k < targetKey),
-            take(pageSize)
-        );
-
-        // make iterators into concrete arrays
-        const top = Array.from(up(asc));
-        const bottom = Array.from(down(desc));
-
-        // startKey is the closest key in the result set to the requested targetKey
-        let startKey;
-        if (matchingValue !== undefined) {
-            startKey = getKey(matchingValue);
-        } else {
-            const topKey = firstEntryKey(top);
-            const bottomKey = firstEntryKey(bottom);
-            startKey = findClosestKey(targetKey, topKey, bottomKey);
-        }
-
-        // assemble contents array by combining the results
-        const match = matchingValue ? [[targetKey, matchingValue]] : [];
-        const contentEntries = [...top.reverse(), ...match, ...bottom];
-        const contentList = contentEntries.map(([k, v]) => v);
-
-        // reverse order for ascending content
-        const contents = keyDirection == "desc" ? contentList : contentList.reverse();
-
-        return { contents, startKey, targetKey };
-    };
-};
-
-/**
- * Finds closest of keys by measuring numeric distance from targetKey
- *
- * @param   {String|Number}  targetKey  Key to match
- * @param   {String|Number}  keys       Keys to check
- *
- * @return  {String|Number} Member of keys closest to targetKey numerially
- */
-const findClosestKey = (targetKey, ...keys) => {
-    return keys
-        .filter((val) => val !== undefined)
-        .reduce((result, key) => {
-            if (result !== undefined) {
-                const keyCloser = distance(targetKey, key) < distance(targetKey, result);
-                return keyCloser ? key : result;
-            }
-            return key;
-        }, undefined);
-};
-
-/**
- * Fresh update map, gets built when inputs change
- */
-export const newUpdateMap = () => {
-    return new SkipList();
-};
-
-/**
- * Generates a can function for consuming updates from the pouchdb-live-find.
- * Update variable will either be a big initial chunk of results matching the
- * query or a follow-up incremental change.
- * @param {*} cfg
- */
-export const processContentUpdate = (cfg = {}) => {
-    const {
-        // function to extract a key from the doc.
-        // this should return a hid or an element_index
-        getKey = (doc) => doc._id,
-    } = cfg;
-
-    return (resultMap, update) => {
-        const { initialMatches = [], action, doc } = update;
-
-        // initial load
-        if (action == ACTIONS.INITIAL && initialMatches.length) {
-            initialMatches.forEach((match) => {
-                resultMap.upsert(getKey(match), match);
-            });
-        }
-
-        // incremental updates
-        if (action && doc) {
-            const key = getKey(doc);
-            switch (action) {
-                case ACTIONS.IGNORE:
-                    console.log("ignored update", doc);
-                    break;
-
-                case ACTIONS.UPDATE:
-                case ACTIONS.ADD:
-                    resultMap.upsert(key, doc);
-                    break;
-
-                case ACTIONS.REMOVE:
-                    resultMap.delete(key);
-                    break;
-            }
-        }
-
-        return resultMap;
-    };
-};
-
-// Configurable function returns a key for the updateMap from a doc during upate operations
-// aggregate results in a map for each set of id + params
-export const getKeyForUpdateMap = (keyField) => (doc) => {
-    let rawKey;
-    if (keyField in doc) {
-        rawKey = doc[keyField];
-    } else {
-        // won't have a full doc on a delete, but it's embeedded in the id
-        const parts = doc._id.split("-");
-        rawKey = parseInt(parts[1]);
-    }
-
-    const result = parseInt(rawKey);
-    if (isNaN(result)) {
-        console.warn("Non integer key for document", doc);
-        throw new Error("Key for update map should be an integer");
-    }
-
-    return result;
-};

client/src/components/History/test/json/collectionContent.json

@@ -1,54 +1,842 @@
 [
     {
-        "element_identifier": "M117-bl.fq",
-        "object": {
-            "contents_url": "/api/dataset_collections/5a1cff6882ddb5b2/contents/fb85969571388350",
-            "collection_type": "paired",
-            "model_class": "DatasetCollection",
-            "id": "fb85969571388350"
-        },
-        "id": "5a9381f02a7b89af",
+        "model_class": "DatasetCollectionElement",
+        "id": "068bb135123e39c3",
+        "element_type": "hda",
         "element_index": 0,
-        "element_type": "dataset_collection",
-        "model_class": "DatasetCollectionElement"
+        "element_identifier": "0",
+        "object": {
+            "id": "e746d19c9c8cc71c",
+            "model_class": "HistoryDatasetAssociation",
+            "state": "ok",
+            "hda_ldda": "hda",
+            "history_id": "e90facaf332c3038"
+        }
     },
     {
-        "element_identifier": "M117-ch.fq",
-        "object": {
-            "contents_url": "/api/dataset_collections/5a1cff6882ddb5b2/contents/ba751ee0539fff04",
-            "collection_type": "paired",
-            "model_class": "DatasetCollection",
-            "id": "ba751ee0539fff04"
-        },
-        "id": "8c49be448cfe29bc",
+        "model_class": "DatasetCollectionElement",
+        "id": "cc0e2ba29cc1d8af",
+        "element_type": "hda",
         "element_index": 1,
-        "element_type": "dataset_collection",
-        "model_class": "DatasetCollectionElement"
+        "element_identifier": "1",
+        "object": {
+            "id": "bd2618511bd6d954",
+            "model_class": "HistoryDatasetAssociation",
+            "state": "ok",
+            "hda_ldda": "hda",
+            "history_id": "e90facaf332c3038"
+        }
     },
     {
-        "element_identifier": "M117C1-bl.fq",
-        "object": {
-            "contents_url": "/api/dataset_collections/5a1cff6882ddb5b2/contents/63cd3858d057a6d1",
-            "collection_type": "paired",
-            "model_class": "DatasetCollection",
-            "id": "63cd3858d057a6d1"
-        },
-        "id": "b847e822bdc195d0",
+        "model_class": "DatasetCollectionElement",
+        "id": "088766ba409c97ae",
+        "element_type": "hda",
         "element_index": 2,
-        "element_type": "dataset_collection",
-        "model_class": "DatasetCollectionElement"
+        "element_identifier": "10",
+        "object": {
+            "id": "b1aecce06b45adc8",
+            "model_class": "HistoryDatasetAssociation",
+            "state": "ok",
+            "hda_ldda": "hda",
+            "history_id": "e90facaf332c3038"
+        }
     },
     {
-        "element_identifier": "M117C1-ch.fq",
-        "object": {
-            "contents_url": "/api/dataset_collections/5a1cff6882ddb5b2/contents/b472e2eb553fa0d1",
-            "collection_type": "paired",
-            "model_class": "DatasetCollection",
-            "id": "b472e2eb553fa0d1"
-        },
-        "id": "13cea0e6d733b865",
+        "model_class": "DatasetCollectionElement",
+        "id": "e48823c5da673276",
+        "element_type": "hda",
         "element_index": 3,
-        "element_type": "dataset_collection",
-        "model_class": "DatasetCollectionElement"
+        "element_identifier": "100",
+        "object": {
+            "id": "01cda2950d5e63c3",
+            "model_class": "HistoryDatasetAssociation",
+            "state": "ok",
+            "hda_ldda": "hda",
+            "history_id": "e90facaf332c3038"
+        }
+    },
+    {
+        "model_class": "DatasetCollectionElement",
+        "id": "cf17043882bf0911",
+        "element_type": "hda",
+        "element_index": 4,
+        "element_identifier": "1000",
+        "object": {
+            "id": "5f2de0d744e8c295",
+            "model_class": "HistoryDatasetAssociation",
+            "state": "ok",
+            "hda_ldda": "hda",
+            "history_id": "e90facaf332c3038"
+        }
+    },
+    {
+        "model_class": "DatasetCollectionElement",
+        "id": "a088eb6d1a3dbcab",
+        "element_type": "hda",
+        "element_index": 5,
+        "element_identifier": "10000",
+        "object": {
+            "id": "43bce5ef002caad3",
+            "model_class": "HistoryDatasetAssociation",
+            "state": "ok",
+            "hda_ldda": "hda",
+            "history_id": "e90facaf332c3038"
+        }
+    },
+    {
+        "model_class": "DatasetCollectionElement",
+        "id": "35cddac7ff46d42d",
+        "element_type": "hda",
+        "element_index": 6,
+        "element_identifier": "100000",
+        "object": {
+            "id": "5fb23b4e4319a7c5",
+            "model_class": "HistoryDatasetAssociation",
+            "state": "ok",
+            "hda_ldda": "hda",
+            "history_id": "e90facaf332c3038"
+        }
+    },
+    {
+        "model_class": "DatasetCollectionElement",
+        "id": "eac6cd9007b1908f",
+        "element_type": "hda",
+        "element_index": 7,
+        "element_identifier": "100001",
+        "object": {
+            "id": "80233f5f28a6c72f",
+            "model_class": "HistoryDatasetAssociation",
+            "state": "ok",
+            "hda_ldda": "hda",
+            "history_id": "e90facaf332c3038"
+        }
+    },
+    {
+        "model_class": "DatasetCollectionElement",
+        "id": "aa4f9dc4708b54b6",
+        "element_type": "hda",
+        "element_index": 8,
+        "element_identifier": "100002",
+        "object": {
+            "id": "1c5156c6a0363f35",
+            "model_class": "HistoryDatasetAssociation",
+            "state": "ok",
+            "hda_ldda": "hda",
+            "history_id": "e90facaf332c3038"
+        }
+    },
+    {
+        "model_class": "DatasetCollectionElement",
+        "id": "cbd3da2c4f117e14",
+        "element_type": "hda",
+        "element_index": 9,
+        "element_identifier": "100003",
+        "object": {
+            "id": "af091cf96ab275f3",
+            "model_class": "HistoryDatasetAssociation",
+            "state": "ok",
+            "hda_ldda": "hda",
+            "history_id": "e90facaf332c3038"
+        }
+    },
+    {
+        "model_class": "DatasetCollectionElement",
+        "id": "68a588a2a9524bc5",
+        "element_type": "hda",
+        "element_index": 10,
+        "element_identifier": "100004",
+        "object": {
+            "id": "97b2c68eeabafb3d",
+            "model_class": "HistoryDatasetAssociation",
+            "state": "ok",
+            "hda_ldda": "hda",
+            "history_id": "e90facaf332c3038"
+        }
+    },
+    {
+        "model_class": "DatasetCollectionElement",
+        "id": "a58d0afa3c515242",
+        "element_type": "hda",
+        "element_index": 11,
+        "element_identifier": "100005",
+        "object": {
+            "id": "33867d668bd83fde",
+            "model_class": "HistoryDatasetAssociation",
+            "state": "ok",
+            "hda_ldda": "hda",
+            "history_id": "e90facaf332c3038"
+        }
+    },
+    {
+        "model_class": "DatasetCollectionElement",
+        "id": "f0c8870154edf2d6",
+        "element_type": "hda",
+        "element_index": 12,
+        "element_identifier": "100006",
+        "object": {
+            "id": "1914213279532687",
+            "model_class": "HistoryDatasetAssociation",
+            "state": "ok",
+            "hda_ldda": "hda",
+            "history_id": "e90facaf332c3038"
+        }
+    },
+    {
+        "model_class": "DatasetCollectionElement",
+        "id": "018c8c5d62a8df97",
+        "element_type": "hda",
+        "element_index": 13,
+        "element_identifier": "100007",
+        "object": {
+            "id": "713b988311601359",
+            "model_class": "HistoryDatasetAssociation",
+            "state": "ok",
+            "hda_ldda": "hda",
+            "history_id": "e90facaf332c3038"
+        }
+    },
+    {
+        "model_class": "DatasetCollectionElement",
+        "id": "d0c151b99af9360b",
+        "element_type": "hda",
+        "element_index": 14,
+        "element_identifier": "100008",
+        "object": {
+            "id": "f38a73020009daba",
+            "model_class": "HistoryDatasetAssociation",
+            "state": "ok",
+            "hda_ldda": "hda",
+            "history_id": "e90facaf332c3038"
+        }
+    },
+    {
+        "model_class": "DatasetCollectionElement",
+        "id": "29eac6dcbd14f883",
+        "element_type": "hda",
+        "element_index": 15,
+        "element_identifier": "100009",
+        "object": {
+            "id": "622229cf50f04387",
+            "model_class": "HistoryDatasetAssociation",
+            "state": "ok",
+            "hda_ldda": "hda",
+            "history_id": "e90facaf332c3038"
+        }
+    },
+    {
+        "model_class": "DatasetCollectionElement",
+        "id": "a158574d58234118",
+        "element_type": "hda",
+        "element_index": 16,
+        "element_identifier": "10001",
+        "object": {
+            "id": "b68d17846c7fbc83",
+            "model_class": "HistoryDatasetAssociation",
+            "state": "ok",
+            "hda_ldda": "hda",
+            "history_id": "e90facaf332c3038"
+        }
+    },
+    {
+        "model_class": "DatasetCollectionElement",
+        "id": "2b98540a14173828",
+        "element_type": "hda",
+        "element_index": 17,
+        "element_identifier": "100010",
+        "object": {
+            "id": "6eeb1155086b617b",
+            "model_class": "HistoryDatasetAssociation",
+            "state": "ok",
+            "hda_ldda": "hda",
+            "history_id": "e90facaf332c3038"
+        }
+    },
+    {
+        "model_class": "DatasetCollectionElement",
+        "id": "8e114d54b2d1f419",
+        "element_type": "hda",
+        "element_index": 18,
+        "element_identifier": "100011",
+        "object": {
+            "id": "e7746068f68fc864",
+            "model_class": "HistoryDatasetAssociation",
+            "state": "ok",
+            "hda_ldda": "hda",
+            "history_id": "e90facaf332c3038"
+        }
+    },
+    {
+        "model_class": "DatasetCollectionElement",
+        "id": "749fa1ed60051f98",
+        "element_type": "hda",
+        "element_index": 19,
+        "element_identifier": "100012",
+        "object": {
+            "id": "8051d6590bfd64e9",
+            "model_class": "HistoryDatasetAssociation",
+            "state": "ok",
+            "hda_ldda": "hda",
+            "history_id": "e90facaf332c3038"
+        }
+    },
+    {
+        "model_class": "DatasetCollectionElement",
+        "id": "911f2df796ad747c",
+        "element_type": "hda",
+        "element_index": 20,
+        "element_identifier": "100013",
+        "object": {
+            "id": "5cba928dd1a05f48",
+            "model_class": "HistoryDatasetAssociation",
+            "state": "ok",
+            "hda_ldda": "hda",
+            "history_id": "e90facaf332c3038"
+        }
+    },
+    {
+        "model_class": "DatasetCollectionElement",
+        "id": "faa6c009defd91d0",
+        "element_type": "hda",
+        "element_index": 21,
+        "element_identifier": "100014",
+        "object": {
+            "id": "eac6ce67595195ec",
+            "model_class": "HistoryDatasetAssociation",
+            "state": "ok",
+            "hda_ldda": "hda",
+            "history_id": "e90facaf332c3038"
+        }
+    },
+    {
+        "model_class": "DatasetCollectionElement",
+        "id": "6129b0ae6f8527f0",
+        "element_type": "hda",
+        "element_index": 22,
+        "element_identifier": "100015",
+        "object": {
+            "id": "98d427a4631d635a",
+            "model_class": "HistoryDatasetAssociation",
+            "state": "ok",
+            "hda_ldda": "hda",
+            "history_id": "e90facaf332c3038"
+        }
+    },
+    {
+        "model_class": "DatasetCollectionElement",
+        "id": "789438fb3dff9d11",
+        "element_type": "hda",
+        "element_index": 23,
+        "element_identifier": "100016",
+        "object": {
+            "id": "ddf8d1e1eb7b0fb7",
+            "model_class": "HistoryDatasetAssociation",
+            "state": "ok",
+            "hda_ldda": "hda",
+            "history_id": "e90facaf332c3038"
+        }
+    },
+    {
+        "model_class": "DatasetCollectionElement",
+        "id": "a7d7479d06608eca",
+        "element_type": "hda",
+        "element_index": 24,
+        "element_identifier": "100017",
+        "object": {
+            "id": "7083648ea486ebf6",
+            "model_class": "HistoryDatasetAssociation",
+            "state": "ok",
+            "hda_ldda": "hda",
+            "history_id": "e90facaf332c3038"
+        }
+    },
+    {
+        "model_class": "DatasetCollectionElement",
+        "id": "440b7cc14c8c4d06",
+        "element_type": "hda",
+        "element_index": 25,
+        "element_identifier": "100018",
+        "object": {
+            "id": "aaf0f2ebe11ae45c",
+            "model_class": "HistoryDatasetAssociation",
+            "state": "ok",
+            "hda_ldda": "hda",
+            "history_id": "e90facaf332c3038"
+        }
+    },
+    {
+        "model_class": "DatasetCollectionElement",
+        "id": "fb4f28349f5baf2d",
+        "element_type": "hda",
+        "element_index": 26,
+        "element_identifier": "100019",
+        "object": {
+            "id": "c78ca882684b7434",
+            "model_class": "HistoryDatasetAssociation",
+            "state": "ok",
+            "hda_ldda": "hda",
+            "history_id": "e90facaf332c3038"
+        }
+    },
+    {
+        "model_class": "DatasetCollectionElement",
+        "id": "87fd7db76c544224",
+        "element_type": "hda",
+        "element_index": 27,
+        "element_identifier": "10002",
+        "object": {
+            "id": "c9c548cca771bdbc",
+            "model_class": "HistoryDatasetAssociation",
+            "state": "ok",
+            "hda_ldda": "hda",
+            "history_id": "e90facaf332c3038"
+        }
+    },
+    {
+        "model_class": "DatasetCollectionElement",
+        "id": "a9431fe40e66ea34",
+        "element_type": "hda",
+        "element_index": 28,
+        "element_identifier": "100020",
+        "object": {
+            "id": "2fbde361d6a31e7e",
+            "model_class": "HistoryDatasetAssociation",
+            "state": "ok",
+            "hda_ldda": "hda",
+            "history_id": "e90facaf332c3038"
+        }
+    },
+    {
+        "model_class": "DatasetCollectionElement",
+        "id": "43acf2f33287edcb",
+        "element_type": "hda",
+        "element_index": 29,
+        "element_identifier": "100021",
+        "object": {
+            "id": "b73abcd044761674",
+            "model_class": "HistoryDatasetAssociation",
+            "state": "ok",
+            "hda_ldda": "hda",
+            "history_id": "e90facaf332c3038"
+        }
+    },
+    {
+        "model_class": "DatasetCollectionElement",
+        "id": "9a74eaab42e02827",
+        "element_type": "hda",
+        "element_index": 30,
+        "element_identifier": "100022",
+        "object": {
+            "id": "7326e2004b6d8758",
+            "model_class": "HistoryDatasetAssociation",
+            "state": "ok",
+            "hda_ldda": "hda",
+            "history_id": "e90facaf332c3038"
+        }
+    },
+    {
+        "model_class": "DatasetCollectionElement",
+        "id": "6d8e4d965b771ea5",
+        "element_type": "hda",
+        "element_index": 31,
+        "element_identifier": "100023",
+        "object": {
+            "id": "e16a1758d8c32a91",
+            "model_class": "HistoryDatasetAssociation",
+            "state": "ok",
+            "hda_ldda": "hda",
+            "history_id": "e90facaf332c3038"
+        }
+    },
+    {
+        "model_class": "DatasetCollectionElement",
+        "id": "c69237419b0ccecb",
+        "element_type": "hda",
+        "element_index": 32,
+        "element_identifier": "100024",
+        "object": {
+            "id": "16d9d7a5407b90bb",
+            "model_class": "HistoryDatasetAssociation",
+            "state": "ok",
+            "hda_ldda": "hda",
+            "history_id": "e90facaf332c3038"
+        }
+    },
+    {
+        "model_class": "DatasetCollectionElement",
+        "id": "2d09ebff4dc98b05",
+        "element_type": "hda",
+        "element_index": 33,
+        "element_identifier": "100025",
+        "object": {
+            "id": "be6aaa8b33c20354",
+            "model_class": "HistoryDatasetAssociation",
+            "state": "ok",
+            "hda_ldda": "hda",
+            "history_id": "e90facaf332c3038"
+        }
+    },
+    {
+        "model_class": "DatasetCollectionElement",
+        "id": "359f5b7c99fbb2a6",
+        "element_type": "hda",
+        "element_index": 34,
+        "element_identifier": "100026",
+        "object": {
+            "id": "f0d2dd7ce1a0d1d7",
+            "model_class": "HistoryDatasetAssociation",
+            "state": "ok",
+            "hda_ldda": "hda",
+            "history_id": "e90facaf332c3038"
+        }
+    },
+    {
+        "model_class": "DatasetCollectionElement",
+        "id": "890b48f1534ba81c",
+        "element_type": "hda",
+        "element_index": 35,
+        "element_identifier": "100027",
+        "object": {
+            "id": "a4243929ecd1e137",
+            "model_class": "HistoryDatasetAssociation",
+            "state": "ok",
+            "hda_ldda": "hda",
+            "history_id": "e90facaf332c3038"
+        }
+    },
+    {
+        "model_class": "DatasetCollectionElement",
+        "id": "4220951a437ba60e",
+        "element_type": "hda",
+        "element_index": 36,
+        "element_identifier": "100028",
+        "object": {
+            "id": "efdbb5c431ed86c4",
+            "model_class": "HistoryDatasetAssociation",
+            "state": "ok",
+            "hda_ldda": "hda",
+            "history_id": "e90facaf332c3038"
+        }
+    },
+    {
+        "model_class": "DatasetCollectionElement",
+        "id": "f306434746a46cb6",
+        "element_type": "hda",
+        "element_index": 37,
+        "element_identifier": "100029",
+        "object": {
+            "id": "0c7525f276af0cb2",
+            "model_class": "HistoryDatasetAssociation",
+            "state": "ok",
+            "hda_ldda": "hda",
+            "history_id": "e90facaf332c3038"
+        }
+    },
+    {
+        "model_class": "DatasetCollectionElement",
+        "id": "abc747938ed8a600",
+        "element_type": "hda",
+        "element_index": 38,
+        "element_identifier": "10003",
+        "object": {
+            "id": "0032a856b26c744e",
+            "model_class": "HistoryDatasetAssociation",
+            "state": "ok",
+            "hda_ldda": "hda",
+            "history_id": "e90facaf332c3038"
+        }
+    },
+    {
+        "model_class": "DatasetCollectionElement",
+        "id": "63636a455c5b21fa",
+        "element_type": "hda",
+        "element_index": 39,
+        "element_identifier": "100030",
+        "object": {
+            "id": "aa1ea414cbd09e79",
+            "model_class": "HistoryDatasetAssociation",
+            "state": "ok",
+            "hda_ldda": "hda",
+            "history_id": "e90facaf332c3038"
+        }
+    },
+    {
+        "model_class": "DatasetCollectionElement",
+        "id": "e8c3a4e75936ee34",
+        "element_type": "hda",
+        "element_index": 40,
+        "element_identifier": "100031",
+        "object": {
+            "id": "9a0d644f6e3ebb6f",
+            "model_class": "HistoryDatasetAssociation",
+            "state": "ok",
+            "hda_ldda": "hda",
+            "history_id": "e90facaf332c3038"
+        }
+    },
+    {
+        "model_class": "DatasetCollectionElement",
+        "id": "25dce1f99c32a06f",
+        "element_type": "hda",
+        "element_index": 41,
+        "element_identifier": "100032",
+        "object": {
+            "id": "c6d7ec9dd85960fd",
+            "model_class": "HistoryDatasetAssociation",
+            "state": "ok",
+            "hda_ldda": "hda",
+            "history_id": "e90facaf332c3038"
+        }
+    },
+    {
+        "model_class": "DatasetCollectionElement",
+        "id": "0f308c8bc2dba01e",
+        "element_type": "hda",
+        "element_index": 42,
+        "element_identifier": "100033",
+        "object": {
+            "id": "c1e28fb74593e606",
+            "model_class": "HistoryDatasetAssociation",
+            "state": "ok",
+            "hda_ldda": "hda",
+            "history_id": "e90facaf332c3038"
+        }
+    },
+    {
+        "model_class": "DatasetCollectionElement",
+        "id": "134707f5254d3467",
+        "element_type": "hda",
+        "element_index": 43,
+        "element_identifier": "100034",
+        "object": {
+            "id": "edb9e58b575cfb18",
+            "model_class": "HistoryDatasetAssociation",
+            "state": "ok",
+            "hda_ldda": "hda",
+            "history_id": "e90facaf332c3038"
+        }
+    },
+    {
+        "model_class": "DatasetCollectionElement",
+        "id": "af2a47bf25ac31c6",
+        "element_type": "hda",
+        "element_index": 44,
+        "element_identifier": "100035",
+        "object": {
+            "id": "595375e544772627",
+            "model_class": "HistoryDatasetAssociation",
+            "state": "ok",
+            "hda_ldda": "hda",
+            "history_id": "e90facaf332c3038"
+        }
+    },
+    {
+        "model_class": "DatasetCollectionElement",
+        "id": "9b4142aa13cacd8d",
+        "element_type": "hda",
+        "element_index": 45,
+        "element_identifier": "100036",
+        "object": {
+            "id": "d917f9554bc7b276",
+            "model_class": "HistoryDatasetAssociation",
+            "state": "ok",
+            "hda_ldda": "hda",
+            "history_id": "e90facaf332c3038"
+        }
+    },
+    {
+        "model_class": "DatasetCollectionElement",
+        "id": "8732705714d1b967",
+        "element_type": "hda",
+        "element_index": 46,
+        "element_identifier": "100037",
+        "object": {
+            "id": "17ac36cd25e09d75",
+            "model_class": "HistoryDatasetAssociation",
+            "state": "ok",
+            "hda_ldda": "hda",
+            "history_id": "e90facaf332c3038"
+        }
+    },
+    {
+        "model_class": "DatasetCollectionElement",
+        "id": "9faac61c404df8d0",
+        "element_type": "hda",
+        "element_index": 47,
+        "element_identifier": "100038",
+        "object": {
+            "id": "fc587326c65e935c",
+            "model_class": "HistoryDatasetAssociation",
+            "state": "ok",
+            "hda_ldda": "hda",
+            "history_id": "e90facaf332c3038"
+        }
+    },
+    {
+        "model_class": "DatasetCollectionElement",
+        "id": "eec6329884514299",
+        "element_type": "hda",
+        "element_index": 48,
+        "element_identifier": "100039",
+        "object": {
+            "id": "0f1c2717b9554064",
+            "model_class": "HistoryDatasetAssociation",
+            "state": "ok",
+            "hda_ldda": "hda",
+            "history_id": "e90facaf332c3038"
+        }
+    },
+    {
+        "model_class": "DatasetCollectionElement",
+        "id": "0a7c50c76808cb45",
+        "element_type": "hda",
+        "element_index": 49,
+        "element_identifier": "10004",
+        "object": {
+            "id": "175262dbcbd9987f",
+            "model_class": "HistoryDatasetAssociation",
+            "state": "ok",
+            "hda_ldda": "hda",
+            "history_id": "e90facaf332c3038"
+        }
+    },
+    {
+        "model_class": "DatasetCollectionElement",
+        "id": "1ff31d34983529f8",
+        "element_type": "hda",
+        "element_index": 50,
+        "element_identifier": "100040",
+        "object": {
+            "id": "7c3fb968df7e50f1",
+            "model_class": "HistoryDatasetAssociation",
+            "state": "ok",
+            "hda_ldda": "hda",
+            "history_id": "e90facaf332c3038"
+        }
+    },
+    {
+        "model_class": "DatasetCollectionElement",
+        "id": "0af255ba4961216c",
+        "element_type": "hda",
+        "element_index": 51,
+        "element_identifier": "100041",
+        "object": {
+            "id": "cf83f2dcc9f7362b",
+            "model_class": "HistoryDatasetAssociation",
+            "state": "ok",
+            "hda_ldda": "hda",
+            "history_id": "e90facaf332c3038"
+        }
+    },
+    {
+        "model_class": "DatasetCollectionElement",
+        "id": "5be07bbe21af2e4a",
+        "element_type": "hda",
+        "element_index": 52,
+        "element_identifier": "100042",
+        "object": {
+            "id": "4efb7c25e2bb7fa8",
+            "model_class": "HistoryDatasetAssociation",
+            "state": "ok",
+            "hda_ldda": "hda",
+            "history_id": "e90facaf332c3038"
+        }
+    },
+    {
+        "model_class": "DatasetCollectionElement",
+        "id": "8ab1435b8102e81e",
+        "element_type": "hda",
+        "element_index": 53,
+        "element_identifier": "100043",
+        "object": {
+            "id": "defeea39a9509c16",
+            "model_class": "HistoryDatasetAssociation",
+            "state": "ok",
+            "hda_ldda": "hda",
+            "history_id": "e90facaf332c3038"
+        }
+    },
+    {
+        "model_class": "DatasetCollectionElement",
+        "id": "842e3a8e9b827812",
+        "element_type": "hda",
+        "element_index": 54,
+        "element_identifier": "100044",
+        "object": {
+            "id": "5f5b885264688585",
+            "model_class": "HistoryDatasetAssociation",
+            "state": "ok",
+            "hda_ldda": "hda",
+            "history_id": "e90facaf332c3038"
+        }
+    },
+    {
+        "model_class": "DatasetCollectionElement",
+        "id": "a5824a57e59503dc",
+        "element_type": "hda",
+        "element_index": 55,
+        "element_identifier": "100045",
+        "object": {
+            "id": "041689872363985f",
+            "model_class": "HistoryDatasetAssociation",
+            "state": "ok",
+            "hda_ldda": "hda",
+            "history_id": "e90facaf332c3038"
+        }
+    },
+    {
+        "model_class": "DatasetCollectionElement",
+        "id": "616698141b35a61f",
+        "element_type": "hda",
+        "element_index": 56,
+        "element_identifier": "100046",
+        "object": {
+            "id": "2839cb63aea76bb4",
+            "model_class": "HistoryDatasetAssociation",
+            "state": "ok",
+            "hda_ldda": "hda",
+            "history_id": "e90facaf332c3038"
+        }
+    },
+    {
+        "model_class": "DatasetCollectionElement",
+        "id": "d36010717df75f3d",
+        "element_type": "hda",
+        "element_index": 57,
+        "element_identifier": "100047",
+        "object": {
+            "id": "4236bdc00992172f",
+            "model_class": "HistoryDatasetAssociation",
+            "state": "ok",
+            "hda_ldda": "hda",
+            "history_id": "e90facaf332c3038"
+        }
+    },
+    {
+        "model_class": "DatasetCollectionElement",
+        "id": "f5d1282ba4e61ea9",
+        "element_type": "hda",
+        "element_index": 58,
+        "element_identifier": "100048",
+        "object": {
+            "id": "bf8ded787d0c5b53",
+            "model_class": "HistoryDatasetAssociation",
+            "state": "ok",
+            "hda_ldda": "hda",
+            "history_id": "e90facaf332c3038"
+        }
+    },
+    {
+        "model_class": "DatasetCollectionElement",
+        "id": "8dc4bd112d29b27d",
+        "element_type": "hda",
+        "element_index": 59,
+        "element_identifier": "100049",
+        "object": {
+            "id": "023453c935463dba",
+            "model_class": "HistoryDatasetAssociation",
+            "state": "ok",
+            "hda_ldda": "hda",
+            "history_id": "e90facaf332c3038"
+        }
     }
-]
+]

client/src/components/History/test/providerTestHelpers.js

@@ -0,0 +1,43 @@
+import { watchForChange } from "jest/helpers";
+import { defaultPayload } from "../providers/ContentProvider";
+
+/**
+ *  Vue adds a bunch of setters and getters so we need to create our own comparison fn
+ */
+export const sameVueObj = (a, b) => {
+    return JSON.stringify(a) == JSON.stringify(b);
+};
+
+/**
+ * Detects a new emission of a "payload" a standardized object delivered by HistoryContentProvider
+ * and CollectionContentProvider, skips the default value that's emitted when it initialized
+ *
+ * @param   {Object}  config  vm, propName, label, other watchForChange configs
+ * @return  {Promise}         promise that resolves with the changed payload
+ */
+export const payloadChange = async (config = {}) => {
+    const { vm, propName = "payload", label = "payload change", skipDefault = false, ...cfg } = config;
+    const result = await watchForChange({ vm, propName, label, ...cfg });
+
+    // can see how fast the response comes back by monitoring this result
+    // console.log("payloadChange result", JSON.stringify(result.newVal));
+    // chopout the placeholder flag for the default
+    // eslint-disable-next-line no-unused-vars
+    const { placeholder, ...payload } = result.newVal;
+    if (skipDefault && sameVueObj(payload, defaultPayload)) {
+        return await payloadChange(config);
+    }
+
+    return payload;
+};
+
+export const reportPayload = (payload, { indexKey = "hid", label = "payload" } = {}) => {
+    const { contents = [], startKeyIndex, ...resultFields } = payload;
+    const keys = contents.map((o, idx) => {
+        const thisKey = o[indexKey];
+        const keyString = idx == startKeyIndex ? `${thisKey}*` : thisKey;
+        return keyString;
+    });
+    const report = { keys, startKeyIndex, ...resultFields };
+    console.log(label, report);
+};

client/src/components/History/test/testCollection.js

@@ -0,0 +1,25 @@
+// Doctored collection & children for unit tests
+import rawCollection from "./json/DatasetCollection";
+import rawNestedCollection from "./json/DatasetCollection.nested.json";
+import rawCollectionContent from "./json/collectionContent.json";
+import rawNestedCollectionContent from "./json/collectionContent.nested.json";
+
+// Sample collection for CCP tests
+
+const parent_url = "/foo";
+
+export const testCollectionContent = rawCollectionContent.map((props) => ({ ...props, parent_url }));
+
+export const testCollection = Object.assign({}, rawCollection, {
+    contents_url: parent_url,
+    element_count: testCollectionContent.length,
+});
+
+export const testNestedCollection = Object.assign({}, rawNestedCollection, {
+    parent_url: parent_url,
+    contents_url: "/foo/bar",
+    element_count: rawNestedCollectionContent.length,
+});
+
+// a list of element_indexes handy for testing
+export const indices = (list) => list.map(({ element_index, _id, parent_url }) => ({ element_index, _id, parent_url }));

client/src/components/History/test/testHistory.js

@@ -0,0 +1,24 @@
+import { History, SearchParams } from "../model";
+import rawHistory from "./json/History.json";
+import rawHistoryContent from "./json/historyContent.json";
+
+// doctor the sample content
+export const testHistoryContent = rawHistoryContent.map((item) => {
+    item.history_id = rawHistory.id;
+    return item;
+});
+
+// doctor the sample history
+export const testHistory = new History({
+    ...rawHistory,
+    hid_counter: testHistoryContent[0].hid + 1,
+});
+
+// simplified filter simlates server-side complete set
+export const serverContent = (filters = new SearchParams()) => {
+    return testHistoryContent.filter((o) => {
+        if (!filters.showDeleted && o.deleted) return false;
+        if (!filters.showHidden && !o.visible) return false;
+        return true;
+    });
+};

client/src/components/WorkflowInvocationState/providers/monitors.js

@@ -1,9 +1,7 @@
 import { of, race, pipe, concat, iif } from "rxjs";
-import { filter, mergeMap, switchMap, delay, share, repeat, take, takeWhile } from "rxjs/operators";
-import { cacheContent, monitorContentQuery } from "components/History/caching";
-import { singleUpdateResult } from "components/History/caching/db/monitorQuery";
+import { filter, map, mergeMap, switchMap, delay, share, repeat, take, takeWhile } from "rxjs/operators";
+import { cacheContent, monitorContentQuery, loadHistoryContents } from "components/History/caching";
 import { fetchDatasetById, fetchDatasetCollectionById, fetchInvocationStepById } from "./fetch";
-import { loadHistoryContents } from "components/History/caching";
 
 // prettier-ignore
 export const datasetMonitor = (cfg = {}) => {
@@ -49,18 +47,21 @@ const buildPouchRequest = (id, contentType) => {
 const createMonitor = (id, contentType, fetcher, spinUpDelay = 250) => {
     // build the pouchdb/mongo request, which is a selector
     // and limits, offsets, etc
+    const request = buildPouchRequest(id, contentType);
 
     // retrieve changes from cache
-    const changes$ = of(buildPouchRequest(id, contentType)).pipe(
+    const changes$ = of(request).pipe(
         monitorContentQuery(),
-        singleUpdateResult(),
+        // singleUpdateResult(),
+        filter(change => change.match && change.doc),
+        map(change => change.doc),
         share()
     );
 
     // cache results that reflect non-deleted existing data in the cache
     const existing$ = changes$.pipe(
         filter(Boolean),
-        );
+    );
 
     // load and cache dataset from server then switch over to the monitor
     const fetchItem$ = of(id).pipe(

client/src/components/WorkflowInvocationState/providers/monitors.test.js

@@ -1,6 +1,6 @@
 import { of } from "rxjs";
 import { ObserverSpy } from "@hirez_io/observer-spy";
-import * as caching from "components/History/caching/loadHistoryContents";
+import { loadHistoryContents } from "../../History/caching";
 import { monitorHistoryUntilTrue, invocationStepMonitor } from "./monitors";
 import * as fetch from "./fetch";
 
@@ -58,7 +58,7 @@ describe("monitorHistoryUntilTrue", () => {
         stopMonitor = false;
         completed = false;
         mockLoadHistoryContent = of(1, 2, 3);
-        jest.spyOn(caching, 'loadHistoryContents').mockImplementation(() => () => mockLoadHistoryContent);
+        loadHistoryContents.mockImplementation(() => () => mockLoadHistoryContent);
         monitor$ = monitorHistoryUntilTrue(stopFn, historyId, 20);
     });
 

client/src/utils/observable/activity.js

@@ -4,11 +4,12 @@
  */
 
 import { merge } from "rxjs";
-import { mapTo, delay, share } from "rxjs/operators";
+import { mapTo, debounceTime, distinctUntilChanged, publish } from "rxjs/operators";
 
-export const activity = (cfg = {}) => (source) => {
-    const { period = 500 } = cfg;
-    const on = source.pipe(mapTo(true), share());
-    const off = on.pipe(delay(period), mapTo(false));
-    return merge(on, off);
+export const activity = (period = 500) => {
+    return publish((src) => {
+        const on = src.pipe(mapTo(true));
+        const off = src.pipe(debounceTime(period), mapTo(false));
+        return merge(on, off).pipe(distinctUntilChanged());
+    });
 };

client/src/utils/observable/burst.js

@@ -0,0 +1,8 @@
+import { publish, buffer, debounceTime } from "rxjs/operators";
+
+export const burst = (period = 100) => {
+    return publish((src) => {
+        const flush = src.pipe(debounceTime(period));
+        return src.pipe(buffer(flush));
+    });
+};

client/src/utils/observable/chunk.js

@@ -0,0 +1,50 @@
+import { map } from "rxjs/operators";
+
+/**
+ * Bust incoming numeric source values into blocks of designated size.
+ *
+ * @param {*} chunkSize
+ * @param {*} ceil
+ */
+// prettier-ignore
+export const chunk = (chunkSize, ceil = false) => {
+    return map((chunkMe) => {
+        const rawVal = chunkMe / chunkSize;
+        return chunkSize * (ceil ? Math.ceil(rawVal) : Math.floor(rawVal));
+    })
+};
+
+/**
+ * Change one parameter to be an multiple of indicated block size. Used to
+ * regulate the URLs we send to the server so that some will be cached. Pos is
+ * the position in the combined inputs array, and the chunkSize is the size of
+ * the block to break the value into.
+ *
+ * Ex: Source Inputs [ x, y, z, 750 ],
+ *     chunkParam(3, 200)
+ *     Results in [ x, y, z, 600]
+ *
+ * @param {integer} pos Input array parameter number to chunk
+ * @param {integer} chunkSize Size of chunks
+ * @param {Boolean} ceil Math.ceil or Math.floor chunk value
+ */
+// prettier-ignore
+export const chunkParam = (pos, chunkSize, ceil = false) => {
+    return map((inputs) => {
+        const chunkMe = inputs[pos];
+        const rawVal = 1.0 * chunkMe / chunkSize;
+        const chunkedVal = chunkSize * (ceil ? Math.ceil(rawVal) : Math.floor(rawVal));
+        const newInputs = inputs.slice();
+        newInputs[pos] = chunkedVal;
+        return newInputs;
+    })
+}
+
+export const chunkProp = (propName, chunkSize, ceil = false) => {
+    return map((obj) => {
+        const chunkMe = obj[propName];
+        const rawVal = (1.0 * chunkMe) / chunkSize;
+        const chunkedVal = chunkSize * (ceil ? Math.ceil(rawVal) : Math.floor(rawVal));
+        return { ...obj, [propName]: chunkedVal };
+    });
+};

client/src/utils/observable/hydrate.js

@@ -0,0 +1,16 @@
+import { pipe } from "rxjs";
+import { map } from "rxjs/operators";
+
+/**
+ * passing objects into the worker removes class information, Pass in an array
+ * of constructors to be assigned, in order of the input array.
+ */
+// prettier-ignore
+export const hydrate = (constructors = []) => pipe(
+    map((inputs) => {
+        return inputs.map((rawInput, i) => {
+            const C = constructors[i];
+            return C ? new C(rawInput) : rawInput;
+        });
+    })
+)

client/src/utils/observable/index.js

@@ -0,0 +1,21 @@
+export { activity } from "./activity";
+export { burst as debounceBurst } from "./burst";
+export { chunk, chunkParam, chunkProp } from "./chunk";
+export { decay } from "./decay";
+export { firstValueFrom } from "./firstValueFrom";
+export { hydrate } from "./hydrate";
+export { monitorBackboneModel } from "./monitorBackboneModel";
+export { monitorXHR } from "./monitorXHR";
+export { needs } from "./needs";
+export { nth } from "./nth";
+export { shareButDie } from "./shareButDie";
+export { show } from "./show";
+export { singleton } from "./singleton";
+export { throttleDistinct } from "./throttleDistinct";
+export { toggle } from "./toggle";
+export { waitForInit } from "./waitForInit";
+export { whenAny } from "./whenAny";
+export { watchVuexSelector } from "./vuex";
+
+// Do not export rxjsDebugging in the barrel file because it has global consequences
+// export { initSpy } from "./rxjsDebugging";

client/src/utils/observable/needs.js

@@ -0,0 +1,12 @@
+// I'm actually unclear on why I needed this operator to make pouchdb
+// observables work. This bears further investigation. To my knowledge
+// withLatestFrom() does the same thing as this operator, yet this works and
+// withLatestFrom() doesn't when used with pouchdb observable creation.
+
+import { zip, of, pipe } from "rxjs";
+import { concatMap } from "rxjs/operators";
+
+// prettier-ignore
+export const needs = (dep$) => pipe(
+    concatMap((src) => zip(of(src), dep$))
+);

client/src/utils/observable/show.js

@@ -0,0 +1,13 @@
+/**
+ * A conditional tap used for debugging so we don't have to constantly add and remove tap statements
+ * while working on rxjs streams
+ */
+import { tap } from "rxjs/operators";
+
+export const show = (flag = false, fn) => {
+    return tap((val) => {
+        if (flag) {
+            fn(val);
+        }
+    });
+};

client/tests/jest/helpers.js

@@ -10,17 +10,22 @@ import { eventHubPlugin } from "components/plugins/eventHub";
 import BootstrapVue from "bootstrap-vue";
 import Vuex from "vuex";
 
+const defaultComparator = (a, b) => a == b;
+
 // Creates a watcher on the indicated vm/prop for use in testing
-export function watchForChange({ vm, opts, propName, timeout = 1000, label = "" }) {
+export function watchForChange(cfg = {}) {
+    const { vm, opts, propName, comparator = defaultComparator, timeout = 1000 } = cfg;
+
     const start = new Date();
     return new Promise((resolve, reject) => {
         const timeoutID = setTimeout(() => {
-            reject(`${propName} never changed ${label}`);
+            reject(`${propName} never changed`);
         }, timeout);
+
         vm.$watch(
             propName,
             function (newVal, oldVal) {
-                if (newVal !== oldVal) {
+                if (!comparator(newVal, oldVal)) {
                     clearTimeout(timeoutID);
                     const stop = new Date();
                     resolve({ start, stop, propName, elapsed: stop - start, newVal, oldVal });
@@ -31,6 +36,25 @@ export function watchForChange({ vm, opts, propName, timeout = 1000, label = ""
     });
 }
 
+export function watchUntil(vm, isComplete, timeout = 1000) {
+    const start = new Date();
+
+    return new Promise((resolve, reject) => {
+        const timeoutID = setTimeout(() => {
+            if (unwatch) unwatch();
+            reject("watch timed out");
+        }, timeout);
+
+        const unwatch = vm.$watch(isComplete, (isDone) => {
+            if (!isDone) return;
+            clearTimeout(timeoutID);
+            const stop = new Date();
+            if (unwatch) unwatch();
+            resolve({ start, stop, elapsed: stop - start });
+        });
+    });
+}
+
 // formats a console.log to more easily read the passed object
 export const show = (obj) => console.log(JSON.stringify(obj, null, 4));
 
@@ -105,3 +129,20 @@ export function waitForLifecyleEvent(vm, lifecycleHookName, cfg = {}) {
     )
     .toPromise();
 }
+
+// Waits for an observable to emit N times then resolves a promise
+// returns a promise because it's easier to test that way in async jest tests
+export const untilNthEmission = (src$, n = 1, safetyTimeout = 2000) => {
+    return new Promise((resolve, reject) => {
+        let result;
+        // prettier-ignore
+        src$.pipe(
+            take(n), 
+            takeUntil(timer(safetyTimeout))
+        ).subscribe({
+            next: (val) => (result = val),
+            complete: () => resolve(result),
+            error: reject,
+        });
+    });
+};