Retrieval
While the search library gives the ability to query and retrieva data from the available datasets, it does not allow similar queries from workflow results, or the ability to perform join and aggregations queries for visualisation and similar purposes. This library provides this through a more dynamic retrieval configuration, which does not depend on a predetermined common data model.
Retrieval Queries
The user can create and test dataset, file and streaming retrieval queries in a similar way to the search library. However, as mentioned above, the user can also select and retrieve data from workflow results and join different datasets/results based on a shared attribute.
Retrieval Configuration
export interface MongoRetrieval {
datasets: any[];
assetIds?: number[];
fields?: Fields[];
filters?: Filter[];
groupBy?: GroupBy;
join?: Join;
buckets?: Buckets;
sample?: number;
sort?: Sort;
pagination?: Pagination;
}
export interface Fields {
datasetId: number;
fields: string[];
}
export interface Filter {
field: string;
fieldType: 'datetime' | 'integer' | 'double' | 'string';
filterType: 'value' | 'range';
from?: number | string;
to?: number | string;
value?: number | string;
}
export interface GroupBy {
field?: string;
aggregation: Aggregation;
}
export interface Aggregation {
field: string;
function: AggregationFunctions;
}
export enum AggregationFunctions {
SUM = 'sum',
MIN = 'min',
MAX = 'max',
AVG = 'avg',
COUNT = 'count',
SET = 'set',
}
export interface Join {
type: 'union' | 'attribute' | 'merge';
configuration?: AttributeBasedJoinConfig;
}
export interface AttributeBasedJoinConfig {
localDatasetId: number;
foreignDatasetId: number;
localJoinAttribute: string;
foreignJoinAttribute: string;
overridingDataset: number;
}
export interface Buckets {
field: string;
count: number;
}
export interface Sort {
field: string;
order: Order;
}
export interface Pagination {
page: number;
pageSize: number;
}
export enum Order {
ASC = 1,
DESC = -1,
}Supported Operations
Field Selection
Configuration:
{
"datasets": ["orders"],
"fields": [
{
"dataset": "orders",
"fields": ["OrderID","CustomerID", "EmployeeID"]
}
]
}Results:
[
{
"_id": "60c891acfb8265ef77425b19",
"OrderID": 10248,
"CustomerID": "VINET",
"EmployeeID": 5
},
{
"_id": "60c891acfb8265ef77425b1a",
"OrderID": 10248,
"CustomerID": "VINET",
"EmployeeID": 5
},
{
"_id": "60c891acfb8265ef77425b1b",
"OrderID": 10248,
"CustomerID": "VINET",
"EmployeeID": 5
},
{
"_id": "60c891acfb8265ef77425b1c",
"OrderID": 10249,
"CustomerID": "TOMSP",
"EmployeeID": 6
},
...
]Filtering
Configuration:
{
"datasets": ["orders"],
"filters": [
{
"field": "ProductID",
"fieldType": "integer",
"filterType": "value",
"value": 11
},
{
"field": "OrderID",
"fieldType": "integer",
"filterType": "range",
"from": 10250,
"to": 10350
}
]
}Results:
[
{
"_id": "60c891acfb8265ef77425b9a",
"OrderID": 10296,
"CustomerID": "LILAS",
"EmployeeID": 6,
"OrderDate": "1996-09-03T00:00:00.000Z",
"RequiredDate": "1996-10-01T00:00:00.000Z",
"ShippedDate": "1996-09-11T00:00:00.000Z",
"ShipVia": 1,
"Freight": 0.12,
"ShipName": "LILA-Supermercado",
"ShipAddress": "Carrera 52 con Ave. Bolívar #65-98 Llano Largo",
"ShipCity": "Barquisimeto",
"ShipRegion": "Lara",
"ShipPostalCode": "3508",
"ShipCountry": "Venezuela",
"ProductID": 11,
"UnitPrice": 16.8,
"Quantity": 12,
"Discount": 0
},
{
"_id": "60c891acfb8265ef77425beb",
"OrderID": 10327,
"CustomerID": "FOLKO",
"EmployeeID": 2,
"OrderDate": "1996-10-11T00:00:00.000Z",
"RequiredDate": "1996-11-08T00:00:00.000Z",
"ShippedDate": "1996-10-14T00:00:00.000Z",
"ShipVia": 1,
"Freight": 63.36,
"ShipName": "Folk och fä HB",
"ShipAddress": "Åkergatan 24",
"ShipCity": "Bräcke",
"ShipPostalCode": "S-844 67",
"ShipCountry": "Sweden",
"ProductID": 11,
"UnitPrice": 16.8,
"Quantity": 50,
"Discount": 0.2
}
]Sorting
Configuration:
{
"datasets": ["orders"],
"sort": {
"field": "ProductID",
"order": "ASC"
}
}Results:
[
{
"_id": "60c891acfb8265ef77425b7c",
"OrderID": 10285,
"CustomerID": "QUICK",
"EmployeeID": 1,
"OrderDate": "1996-08-20T00:00:00.000Z",
"RequiredDate": "1996-09-17T00:00:00.000Z",
"ShippedDate": "1996-08-26T00:00:00.000Z",
"ShipVia": 2,
"Freight": 76.83,
"ShipName": "QUICK-Stop",
"ShipAddress": "Taucherstraße 10",
"ShipCity": "Cunewalde",
"ShipPostalCode": "01307",
"ShipCountry": "Germany",
"ProductID": 1,
"UnitPrice": 14.4,
"Quantity": 45,
"Discount": 0.2
},
...
]Limit/pagination
Configuration:
{
"datasets": ["orders"],
"pagination": {
"page": 1,
"pageSize": 2
}
}Results:
[
{
"_id": "60c891acfb8265ef77425b19",
"OrderID": 10248,
"CustomerID": "VINET",
"EmployeeID": 5,
"OrderDate": "1996-07-04T00:00:00.000Z",
"RequiredDate": "1996-08-01T00:00:00.000Z",
"ShippedDate": "1996-07-16T00:00:00.000Z",
"ShipVia": 3,
"Freight": 32.38,
"ShipName": "Vins et alcools Chevalier",
"ShipAddress": "59 rue de l'Abbaye",
"ShipCity": "Reims",
"ShipPostalCode": "51100",
"ShipCountry": "France",
"ProductID": 11,
"UnitPrice": 14,
"Quantity": 12,
"Discount": 0
},
{
"_id": "60c891acfb8265ef77425b1a",
"OrderID": 10248,
"CustomerID": "VINET",
"EmployeeID": 5,
"OrderDate": "1996-07-04T00:00:00.000Z",
"RequiredDate": "1996-08-01T00:00:00.000Z",
"ShippedDate": "1996-07-16T00:00:00.000Z",
"ShipVia": 3,
"Freight": 32.38,
"ShipName": "Vins et alcools Chevalier",
"ShipAddress": "59 rue de l'Abbaye",
"ShipCity": "Reims",
"ShipPostalCode": "51100",
"ShipCountry": "France",
"ProductID": 42,
"UnitPrice": 9.8,
"Quantity": 10,
"Discount": 0
}
]Group-by
Configuration:
{
"datasets": ["orders"],
"groupBy": {
"field": "ProductID",
"aggregation": {
"field": "Quantity",
"function": "sum"
}
}
}Results:
[
{
"_id": 62,
"sum": 1083
},
{
"_id": 26,
"sum": 753
},
{
"_id": 63,
"sum": 445
},
{
"_id": 21,
"sum": 1016
},
{
"_id": 76,
"sum": 981
},
{
"_id": 33,
"sum": 755
},
...
]Note: _id is the value of each group of the field in groupBy
Union Join
Configuration:
{
"datasets": ["orders", "products"],
"join": {
"type": "union"
}
}Results:
[
{
"_id": "60c891acfb8265ef77425b19",
"OrderID": 10248,
"CustomerID": "VINET",
"EmployeeID": 5,
"OrderDate": "1996-07-04T00:00:00.000Z",
"RequiredDate": "1996-08-01T00:00:00.000Z",
"ShippedDate": "1996-07-16T00:00:00.000Z",
"ShipVia": 3,
"Freight": 32.38,
"ShipName": "Vins et alcools Chevalier",
"ShipAddress": "59 rue de l'Abbaye",
"ShipCity": "Reims",
"ShipPostalCode": "51100",
"ShipCountry": "France",
"ProductID": 11,
"UnitPrice": 14,
"Quantity": 12,
"Discount": 0
},
....,
{
"_id": "60c89126fb8265ef77425b10",
"ProductID": 77,
"ProductName": "Original Frankfurter grüne Soße",
"SupplierID": 12,
"CategoryID": 2,
"QuantityPerUnit": "12 boxes",
"UnitPrice": 13,
"UnitsInStock": 32,
"UnitsOnOrder": 0,
"ReorderLevel": 15,
"Discontinued": false
}
]Attribute-based Join (left-join)
Configuration:
{
"datasets": ["orders","prducts"],
"join": {
"type": "attribute",
"configuration": {
"localDatasetId": "orders",
"foreignDatasetId": "products",
"localJoinAttribute": "ProductID",
"foreignJoinAttribute":"ProductID"
}
}
}Results:
[
{
"orders": {
"_id": "60c891acfb8265ef77425b19",
"OrderID": 10248,
"CustomerID": "VINET",
"EmployeeID": 5,
"OrderDate": "1996-07-04T00:00:00.000Z",
"RequiredDate": "1996-08-01T00:00:00.000Z",
"ShippedDate": "1996-07-16T00:00:00.000Z",
"ShipVia": 3,
"Freight": 32.38,
"ShipName": "Vins et alcools Chevalier",
"ShipAddress": "59 rue de l'Abbaye",
"ShipCity": "Reims",
"ShipPostalCode": "51100",
"ShipCountry": "France",
"ProductID": 11,
"UnitPrice": 14,
"Quantity": 12,
"Discount": 0
},
"products": {
"_id": "60c89126fb8265ef77425ace",
"ProductID": 11,
"ProductName": "Queso Cabrales",
"SupplierID": 5,
"CategoryID": 4,
"QuantityPerUnit": "1 kg pkg.",
"UnitPrice": 21,
"UnitsInStock": 22,
"UnitsOnOrder": 30,
"ReorderLevel": 30,
"Discontinued": false
}
},
...
]Sampling
Configuration:
{
"datasets": ["orders"],
"sample": 1000
}Results:
[
{
"_id": "60c891acfb8265ef77425de6",
"OrderID": 10519,
"CustomerID": "CHOPS",
"EmployeeID": 6,
"OrderDate": "1997-04-28T00:00:00.000Z",
"RequiredDate": "1997-05-26T00:00:00.000Z",
"ShippedDate": "1997-05-01T00:00:00.000Z",
"ShipVia": 3,
"Freight": 91.76,
"ShipName": "Chop-suey Chinese",
"ShipAddress": "Hauptstr. 31",
"ShipCity": "Bern",
"ShipPostalCode": "3012",
"ShipCountry": "Switzerland",
"ProductID": 60,
"UnitPrice": 34,
"Quantity": 10,
"Discount": 0.05
}
....
]Auto buckets
Configuration:
{
"datasets": ["orders"],
"buckets": {
"field": "OrderDate",
"count": 5
}
}Results:
[
{
"_id": {
"min": "1996-07-04T00:00:00.000Z",
"max": "1997-01-09T00:00:00.000Z"
},
"count": 431
},
{
"_id": {
"min": "1997-01-09T00:00:00.000Z",
"max": "1997-06-18T00:00:00.000Z"
},
"count": 433
},
{
"_id": {
"min": "1997-06-18T00:00:00.000Z",
"max": "1997-11-14T00:00:00.000Z"
},
"count": 434
},
{
"_id": {
"min": "1997-11-14T00:00:00.000Z",
"max": "1998-02-27T00:00:00.000Z"
},
"count": 433
},
{
"_id": {
"min": "1998-02-27T00:00:00.000Z",
"max": "1998-05-06T00:00:00.000Z"
},
"count": 424
}
]