haro
Harō is a modern immutable DataStore built with ES6 features, which can be wired to an API for a complete feedback loop.
It is un-opinionated, and offers a plug'n'play solution to modeling, searching, & managing data on the client, or server
(in RAM). It is a partially persistent data structure, by maintaining version sets of records in versions
(MVCC).
Synchronous commands return instantly (Array
or Tuple
), while asynchronous commands return Promises
which will
resolve or reject in the future. This allows you to build complex applications without worrying about managing async
code.
Harō indexes have the following structure Map (field/property) > Map (value) > Set (PKs)
which allow for quick & easy
searching, as well as inspection. Indexes can be managed independently of del()
& set()
operations, for example you
can lazily create new indexes via reindex(field)
, or sortBy(field)
.
Requirements
Harō is built with ES6+ features, and requires polyfills for ES5 or earlier environments.
Map
Set
Promise
fetch()
-
deferred()
see tiny-defer for loading in a browser -
tuple()
see tiny-tuple for loading in a browser
How to use
Harō takes two optional arguments, the first is an Array
of records to set asynchronously, & the second is a
configuration descriptor.
var storeDefaults = haro();
var storeRecords = haro([{name: 'John Doe', age: 30}, {name: 'Jane Doe', age: 28}]);
var storeCustom = haro(null, {key: 'id'});
Persistent Storage
Harō is an in RAM only DataStore, so state could be lost if your program unexpectedly restarted, or some kind of machine failure were to occur. To handle this serious problem, Harō affords a 1-n relationship with persistent storage adapters. You can register one or many adapters, and data updates will asynchronously persist to the various long term storage systems.
DataStore records will be stored separate of the DataStore snapshot itself (if you decide to leverage it), meaning you
are responsible for doing a load()
& save()
at startup & shutdown. This is a manual process because it could be a
time bottleneck in the middle of using your application. Loading an individual record will update the DataStore with
value from persistent storage.
DataStore snapshots & individual records can be removed from persistent storage with unload()
; it is not recommended
to do this for an individual record, and to instead rely on del()
, but it's afforded because it may be required.
Creating an Adapter
Adapters are simple in nature (can be isomorphic), and pretty easy to create! Follow the template below, fill in the
gaps for your adapter as needed, such as handling multiple connection pools, etc.. The input parameters should not be
mutated. The return must be a Promise
.
"use strict";
const deferred = require("tiny-defer");
function adapter (store, op, key, data) {
let defer = deferred(),
record = key !== undefined,
config = store.adapters.myAdapterName,
prefix = config.prefix || store.id,
lkey = prefix + (record ? "_" + key : "")),
client = "Your driver instance";
if (op === "get") {
client.get(lkey, function (e, reply) {
let result = JSON.parse(reply || null);
if (e) {
defer.reject(e);
} else if (result) {
defer.resolve(result);
} else if (record) {
defer.reject(new Error("Record not found in myAdapterName"));
} else {
defer.reject([]);
}
});
} else if (op === "remove") {
client.del(lkey, function (e) {
if (e) {
defer.reject(e);
} else {
defer.resolve(true);
}
});
} else if (op === "set") {
client.set(lkey, JSON.stringify(record ? data : store.toArray()), function (e) {
if (e) {
defer.reject(e);
} else {
defer.resolve(true);
}
});
}
return defer.promise;
}
module.exports = adapter;
Examples
Piping Promises
var store = haro();
console.log(store.total); // 0
store.set(null, {abc: true}).then(function (arg) {
console.log(arg); // [$uuid, {abc: true}];
console.log(store.total); // 1
return store.set(arg[0], {abc: false});
}).then(function (arg) {
console.log(arg); // [$uuid, {abc: false}];
console.log(store.versions.get(arg[0]).size); // 1;
return store.del(arg[0])
}).then(function () {
console.log(store.total); // 0;
}).catch(function (e) {
console.error(e.stack || e.message || e);
});
Indexes & Searching
var store = haro(null, {index: ['name', 'age']}),
data = [{name: 'John Doe', age: 30}, {name: 'Jane Doe', age: 28}];
store.batch(data, 'set').then(function (records) {
console.log(records[0]); // [$uuid, {name: 'John Doe', age: 30}]
console.log(store.total); // 2
console.log(store.find({age: 28})); // [[$uuid, {name: 'Jane Doe', age: 28}]]
console.log(store.search(/^ja/i, 'name')); // [[$uuid, {name: 'Jane Doe', age: 28}]]
console.log(store.search(function (age) { return age < 30; }, 'age')); // [[$uuid, {name: 'Jane Doe', age: 28}]]
}).catch(function (e) {
console.error(e.stack || e.message || e);
});
MVCC versioning
var store = haro();
store.set(null, {abc: true}).then(function (arg) {
return store.set(arg[0], {abc: false});
}).then(function (arg) {
return store.set(arg[0], {abc: true});
}).then(function (arg) {
store.versions.get(arg[0]).forEach(function (i) { console.log(i[0]); }); // {abc: true}, {abc: false}
}).catch(function (e) {
console.error(e.stack || e.message || e);
});
Benchmarked
A benchmark is included in the repository, and is useful for gauging how haro will perform on different hardware, & software.
Please consider that batch()
, & set()
use Promises
and incur time as a cost. The following results are from an Apple
MacBook Air (Early 2014) / 8GB RAM / 512GB SSD / OS X Yosemite:
time to load data: 523.421068ms
datastore record count: 15000
name indexes: 15000
testing time to 'find()' a record (first one is cold):
0.31272ms
0.123786ms
0.051086ms
0.053974ms
0.045515ms
testing time to 'search(regex, index)' for a record (first one is cold):
2.676046ms
1.760155ms
2.087627ms
1.558766ms
1.568192ms
Configuration
adapters Object
Object of {(storage): (connection string)} pairs. Collection/table name is the value of this.id
.
Available adapters: mongo
Example of specifying MongoDB as persistent storage:
var store = haro(null, {
adapters: {
mongo: "mongo://localhost/mine"
}
});
config Object
Default settings for fetch()
.
Example of specifying a bearer token authorization header:
var store = haro(null, {
config: {
headers: {
authorization: 'Bearer abcdef'
}
});
index Array
Array of values to index. Composite indexes are supported, by using the default delimiter (this.delimiter
).
Non-matches within composites result in blank values.
Example of fields/properties to index:
var store = haro(null, {index: ['field1', 'field2', 'field1|field2|field3']);
key String
Optional Object
key to utilize as Map
key, defaults to a version 4 UUID
if not specified, or found.
Example of specifying the primary key:
var store = haro(null, {key: 'field'});
logging Boolean
Logs persistent storage messages to console
, default is true
.
onbatch Function
Event listener for a batch operation, receives two arguments ['type', Tuple
].
onclear Function
Event listener for clearing the data store.
ondelete Function
Event listener for when a record is deleted, receives the record key.
onerror Function
Event listener for errors which occur during common operations, receives two arguments ['type', Error
]
onset Function
Event listener for when a record is set, receives a Tuple
.
onsync Function
Event listener for synchronizing with an API, receives a Tuple
of Tuples
.
source String
Optional Object
key to retrieve data from API responses, see setUri()
.
Example of specifying the source of data:
var store = haro(null, {source: 'data'});
versioning Boolean
Enable/disable MVCC style versioning of records, default is true
. Versions are stored in Sets
for easy iteration.
Example of disabling versioning:
var store = haro(null, {versioning: false});
Properties
data Map
Map
of records, updated by del()
& set()
.
indexes Map
Map of indexes, which are Sets containing Map keys.
patch Boolean
Set from the success handler of sync()
, infers PATCH
requests are supported by the API collection.
registry Array
Array representing the order of this.data
.
total Number
Total records in the DataStore.
uri String
API collection URI the DataStore is wired to, in a feedback loop (do not modify, use setUri()
). Setting the value creates an implicit relationship with records, e.g. setting /users
would imply a URI structure of /users/{key}
. Trailing slashes may be stripped.
versions Map
Map
of Sets
of records, updated by set()
.
API
batch(array, type) Promise
The first argument must be an Array
, and the second argument must be del
or set
. Batch operations with a DataStore
that is wired to an API with pagination enabled & PATCH
support may create erroneous operations, such as add
where
replace
is appropriate; this will happen because the DataStore will not have the entire data set to generate it's
JSONPatch request.
var haro = require('haro'),
store = haro(null, {key: 'id', index: ['name']}),
i = -1,
nth = 100,
data = [];
while (++i < nth) {
data.push({id: i, name: 'John Doe' + i});
}
store.batch(data, 'set').then(function(records) {
// records is a Tuple of Tuples
}, function (e) {
console.error(e.stack);
});
clear() self
Removes all key/value pairs from the DataStore.
Example of clearing a DataStore:
var store = haro();
// Data is added
store.clear();
del(key) Promise
Deletes the record.
Example of deleting a record:
var store = haro();
store.set(null, {abc: true}).then(function (rec) {
return store.del(rec[0]);
}, function (e) {
throw e;
}).then(function () {
console.log(store.total); // 0
}, function (e) {
console.error(e.stack);
});
dump(type="records") Array or Object
Returns the records or indexes of the DataStore as mutable Array
or Object
, for the intention of reuse/persistent storage without relying on an adapter which would break up the data set.
var store = haro();
// Data is loaded
var records = store.dump();
var indexes = store.dump('indexes');
// Save records & indexes
entries() MapIterator
Returns returns a new Iterator
object that contains an array of [key, value]
for each element in the Map
object in
insertion order.
Example of deleting a record:
var store = haro(),
item, iterator;
// Data is added
iterator = store.entries();
item = iterator.next();
do {
console.log(item.value);
item = iterator.next();
} while (!item.done);
filter(callbackFn) Tuple
Returns a Tuple
of double Tuples
with the shape [key, value]
for records which returned true
to
callbackFn(value, key)
.
Example of filtering a DataStore:
var store = haro();
// Data is added
store.filter(function (value) {
return value.something === true;
});
find(where) Tuple
Returns a Tuple
of double Tuples
with found by indexed values matching the where
.
Example of finding a record(s) with an identity match:
var store = haro(null, {index: ['field1']});
// Data is added
store.find({field1: 'some value'});
forEach(callbackFn[, thisArg]) Undefined
Calls callbackFn
once for each key-value pair present in the Map
object, in insertion order. If a thisArg
parameter is provided to forEach
, it will be used as the this value for each callback.
Example of deleting a record:
var store = haro();
store.set(null, {abc: true}).then(function (rec) {
store.forEach(function (value, key) {
console.log(key);
});
}, function (e) {
console.error(e.stack);
});
get(key) Tuple
Gets the record as a double Tuple
with the shape [key, value]
.
Example of getting a record with a known primary key value:
var store = haro();
// Data is added
store.get('keyValue');
join(other, on[, type="inner", where=[]]) Array
Joins this
instance of Haro
with another, on a field/property. Supports "inner", "left", & "right" JOINs. Resulting
composite records implement a storeId_field
convention for fields/properties. The optional forth parameter is an Array
which can be used for WHERE clauses, similar to find()
, [store1, store2]
.
var store1 = haro([{id: "abc", name: "jason", age: 35}, {id: "def", name: "jen", age: 31}], {id: 'users', key: 'id', index: ['name', 'age']});
var store2 = haro([{id: 'ghi', user: "abc", value: 40}], {id: 'values', key: 'id', index: ['user', 'value']});
// Join results
store1.join(store2, "user", "inner").then(function (records) {
console.log(records);
// [{"users_id":"abc","users_name":"jason","users_age":35,"values_id":"ghi","values_user":"abc","values_value":40}]
}, function (e) {
console.error(e.stack || e.message || e);
});
store1.join(store2, "user", "inner", [{age: 31}]).then(function (records) {
console.log(records);
// []
}, function (e) {
console.error(e.stack || e.message || e);
});
store1.join(store2, "user", "left").then(function (records) {
console.log(records);
// [{"users_id":"abc","users_name":"jason","users_age":35,"values_id":"ghi","values_user":"abc","values_value":40},
// {"users_id":"def","users_name":"jen","users_age":31,"values_id":null,"values_user":null,"values_value":null}]
}, function (e) {
console.error(e.stack || e.message || e);
});
store1.join(store2, "user", "right").then(function (records) {
console.log(records);
// [{"values_id":"ghi","values_user":"abc","values_value":40,"users_id":"abc","users_name":"jason","users_age":35}]
}, function (e) {
console.error(e.stack || e.message || e);
});
keys() MapIterator
Returns a new Iterator
object that contains the keys for each element in the Map
object in insertion order.`
Example of getting an iterator, and logging the results:
var store = haro(),
item, iterator;
// Data is added
iterator = store.keys();
item = iterator.next();
do {
console.log(item.value);
item = iterator.next();
} while (!item.done);
limit(max, offset=0) Tuple
Returns a Tuple
of double Tuples
with the shape [key, value]
for the corresponding range of records.
Example of paginating a data set:
var store = haro(), ds1, ds2;
// Data is added
console.log(store.total); // >10
ds1 = store.limit(10); // [0-9]
ds2 = store.limit(10, 10); // [10-19]
console.log(ds1.length === ds2.length); // true
console.log(JSON.stringify(ds1[0][1]) === JSON.stringify(ds2[0][1])); // false
load([adapter="mongo", key]) Promise
Loads the DataStore, or a record from a specific persistent storage & updates the DataStore. The DataStore will be cleared
prior to loading if key
is omitted.
map(callbackFn) Tuple
Returns a Tuple
of the returns of callbackFn(value, key)
.
Example of mapping a DataStore:
var store = haro();
// Data is added
store.map(function (value) {
return value.property;
});
offload(data[, cmd="index", index=this.index]) Promise
Returns a Promise
for an offloaded work load, such as preparing indexes in a Worker
. This method is ideal for dealing
with large data sets which could block a UI thread. This method requires Blob
& Worker
.
Example of offloading index creation:
var store = haro(null, {index: ['name', 'age'], key: 'guid'}),
data = [{guid: 'abc', name: 'Jason Mulligan', age: 35}];
store.offload(data).then(function (args) {
store.override(data);
store.override(args, 'indexes');
}, function (e) {
console.error(e);
});
override(data[, type="records", fn]) Promise
Returns a Promise
for the new state. This is meant to be used in a paired override of the indexes & records, such that
you can avoid the Promise
based code path of a batch()
insert or load()
. Accepts an optional third parameter to perform the
transformation to simplify cross domain issues.
Example of overriding a DataStore:
var store = haro();
store.override({'field': {'value': ['pk']}}, "indexes").then(function () {
// Indexes have been overridden, no records though! override as well?
}, function (e) {
console.error(e.stack);
});
reindex([index]) Haro
Re-indexes the DataStore, to be called if changing the value of index
.
Example of mapping a DataStore:
var store = haro();
// Data is added
// Creating a late index
store.index('field3');
// Recreating indexes, this should only happen if the store is out of sync caused by developer code.
store.index();
register(key, fn) Haro
Registers a persistent storage adapter.
Example of registering an adapter:
var haro = require('haro'),
store;
// Configure a store to utilize the adapter
store = haro(null, {
adapters: {
mongo: "mongo://localhost/mydb"
}
});
// Register the adapter
store.register('mongo', require('haro-mongo'));
request(input, config) Promise
Returns a Promise
for a fetch()
with a triple Tuple
[body
, status
, headers
] as the resolve()
& reject()
argument.
Example of mapping a DataStore:
var store = haro();
store.request('https://somedomain.com/api').then(function (arg) {
console.log(arg); // [body, status, headers]
}, function (arg) {
console.error(arg[0]);
});
save([adapter]) Promise
Saves the DataStore to persistent storage.
search(arg[, index=this.index]) Tuple
Returns a Tuple
of double Tuples
with the shape [key, value]
of records found matching arg
.
If arg
is a Function
(parameters are value
& index
) a match is made if the result is true
, if arg
is a RegExp
the field value must .test()
as true
, else the value must be an identity match. The index
parameter can be a String
or Array
of Strings
;
if not supplied it defaults to this.index
.
Example of searching with a predicate function:
var store = haro(null, {index: ['name', 'age']}),
data = [{name: 'John Doe', age: 30}, {name: 'Jane Doe', age: 28}];
store.batch(data, 'set').then(function (records) {
console.log(store.search(function (age) {
return age < 30;
}, 'age')); // [[$uuid, {name: 'Jane Doe', age: 28}]]
}, function (e) {
console.error(e.stack || e.message || e);
});
set(key, data, batch=false, override=false) Promise
Returns a Promise
for setting/amending a record in the DataStore, if key
is false
a version 4 UUID
will be
generated.
If override
is true
, the existing record will be replaced instead of amended.
Example of creating a record:
var store = haro(null, {key: 'id'});
store.set(null, {id: 1, name: 'John Doe'}).then(function (record) {
console.log(record); // [1, {id: 1, name: 'Jane Doe'}]
}, function (e) {
console.error(e.stack || e.message || e);
});
setUri(uri, clear=false) Promise
Returns a Promise
for wiring the DataStore to an API, with the retrieved record set as the resolve()
argument. This
creates an implicit mapping of $uri/{key}
for records.
Pagination can be implemented by conditionally supplying true
as the second argument. Doing so will clear()
the
DataStore prior to a batch insertion.
If PATCH
requests are supported by the collection batch()
, del()
& set()
will make JSONPatch
requests. If a
405
/ Method not Allowed
response occurs from a PATCH
request, the DataStore will fallback to the appropriate
method & disable PATCH
for subsequent requests.
Example setting the URI of the DataStore:
var store = haro(null, {key: 'id'});
store.setUri('https://api.somedomain.com').then(function (records) {
console.log(records); // [[$id, {...}], ...]
}, function (arg) {
console.error(arg[0]); // [body, statusCode]
});
Example of pagination, by specifying clear
:
var store = haro(null, {key: 'id'});
store.setUri('https://api.somedomain.com?page=1').then(function (records) {
console.log(records); // [[$id, {...}], ...]
}, function (arg) {
console.log(arg[0]); // [body, statusCode]
});
// Later, based on user interaction, change the page
store.setUri('https://api.somedomain.com?page=2', true).then(function (records) {
console.log(records); // [[$id, {...}], ...]
}, function (arg) {
console.error(arg[0]); // [body, statusCode]
});
sort(callbackFn, [frozen = true]) Array
Returns an Array of the DataStore, sorted by callbackFn
.
Example of sorting like an Array
:
var store = haro(null, {index: ['name', 'age']}),
data = [{name: 'John Doe', age: 30}, {name: 'Jane Doe', age: 28}];
store.batch(data, 'set').then(function (records) {
console.log(store.sort(function (a, b) {
return a < b ? -1 : (a > b ? 1 : 0);
})); // [{name: 'Jane Doe', age: 28}, {name: 'John Doe', age: 30}]
}, function (e) {
console.error(e.stack || e.message || e);
});
sortBy(index) Tuple
Returns a Tuple
of double Tuples
with the shape [key, value]
of records sorted by an index.
Example of sorting by an index:
var store = haro(null, {index: ['name', 'age']}),
data = [{name: 'John Doe', age: 30}, {name: 'Jane Doe', age: 28}];
store.batch(data, 'set').then(function (records) {
console.log(store.sortBy('age')); // [[$uuid, {name: 'Jane Doe', age: 28}], [$uuid, {name: 'John Doe', age: 30}]]
}, function (e) {
console.error(e.stack || e.message || e);
});
sync(clear=false) Promise
Synchronises the DataStore with an API collection. If clear
is true
, the DataStore will have clear()
executed
prior to batch()
upon a successful retrieval of data.
Example of sorting by an index:
var store = haro(null, {key: 'id'}),
interval;
store.setUri('https://api.somedomain.com').then(function (records) {
console.log(records); // [[$id, {...}], ...]
}, function (arg) {
console.error(arg[0]); // [body, statusCode]
});
// Synchronizing the store every minute
interval = setInterval(function () {
store.sync();
}, 60000);
toArray([data, freeze=true]) Array
Returns an Array of the DataStore, or a subset.
Example of casting to an Array
:
var store = haro(),
data = [{name: 'John Doe', age: 30}, {name: 'Jane Doe', age: 28}];
store.batch(data, 'set').then(function (records) {
console.log(store.toArray()); // [{name: 'John Doe', age: 30}, {name: 'Jane Doe', age: 28}]
console.log(store.toArray(store.limit(1))); // [{name: 'John Doe', age: 30}]
}, function (e) {
console.error(e.stack || e.message || e);
});
toObject([data, freeze=true]) Object
Returns an Object of the DataStore.
Example of casting to an Object
:
var store = haro(null, {key: 'guid'}),
data = [{guid: 'abc', name: 'John Doe', age: 30}, {guid: 'def', name: 'Jane Doe', age: 28}];
store.batch(data, 'set').then(function (records) {
console.log(store.toObject()); // {abc: {guid: 'abc', name: 'John Doe', age: 30}, def: {guid: 'def', name: 'Jane Doe', age: 28}}
console.log(store.toObject(store.limit(1)); // {abc: {guid: 'abc', name: 'John Doe', age: 30}}}
}, function (e) {
console.error(e.stack || e.message || e);
});
transform(input[, fn]) Mixed
Transforms Map
to Object
, Object
to Map
, Set
to Array
, & Array
to Set
. Accepts an optional second parameter to perform the
transformation to simplify cross domain issues.
haro.transform()
is exposed so that you can either duplicate it into the current context with toString()
&
new Function()
, or simply re-implement, for situations where you need to supply the transformation Function
.
var store = haro(null, {key: 'guid', index: ['name'}),
data = [{guid: 'abc', name: 'John Doe', age: 30}, {guid: 'def', name: 'Jane Doe', age: 28}];
store.batch(data, 'set').then(function (records) {
console.log(store.transform(store.indexes)); // {age: {'28': ['def'], '30': ['abc']}, name: {'John Doe': ['abc'], 'Jane Doe': ['def']}}
}, function (e) {
console.error(e.stack || e.message || e);
});
unload([adapter=mongo, key]) Promise
Unloads the DataStore, or a record from a specific persistent storage (delete).
unregister(key) Haro
Un-registers a persistent storage adapter.
Example of unregistering an adapter:
var haro = require('haro'),
store;
// Register the adapter
haro.register('mongo', require('haro-mongo'));
// Configure a store to utilize the adapter
store = haro(null, {
adapters: {
mongo: "mongo://localhost/mydb"
}
});
// Later...
store.unregister('haro');
values() MapIterator
Returns a new Iterator
object that contains the values for each element in the Map
object in insertion order.
Example of iterating the values:
var store = haro(),
data = [{name: 'John Doe', age: 30}, {name: 'Jane Doe', age: 28}];
store.batch(data, 'set').then(function (records) {
var iterator = store.values(),
item = iterator.next();
do {
console.log(item.value);
item = iterator.next();
} while (!item.done);
}, function (e) {
console.error(e.stack || e.message || e);
});
License
Copyright (c) 2015 Jason Mulligan Licensed under the BSD-3 license