From 86238e9869f3fad518fb5aaf53e072a4109a1024 Mon Sep 17 00:00:00 2001 From: Hamza Aamer Date: Wed, 11 Nov 2026 23:59:54 +0100 Subject: [PATCH] Audio and video recordings in datasets - uploadMedia: add a recording to a dataset from a File/Blob, a Buffer or ArrayBuffer, or a file path in Node.js - sendDataset takes recordings as media, also without time-series - listDatasets and downloadMedia --- README.md | 37 +++++++++ __tests__/test_exports.js | 2 +- __tests__/test_media.js | 161 ++++++++++++++++++++++++++++++++++++++ dist/index.browser.esm.js | 140 ++++++++++++++++++++++++++++++--- dist/index.browser.js | 138 +++++++++++++++++++++++++++++--- dist/index.js | 138 +++++++++++++++++++++++++++++--- dist/index.mjs | 140 ++++++++++++++++++++++++++++++--- index.d.ts | 84 +++++++++++++++++++- rollup.config.mjs | 2 +- src/http.js | 8 +- src/index.js | 36 +++++++-- src/media.js | 101 ++++++++++++++++++++++++ 12 files changed, 932 insertions(+), 55 deletions(-) create mode 100644 __tests__/test_media.js create mode 100644 src/media.js diff --git a/README.md b/README.md index f867f10..2571013 100644 --- a/README.md +++ b/README.md @@ -108,6 +108,37 @@ const datasetId = await edgeML.sendDataset("backendUrl", "deviceApiKey", { }); ``` +### Audio and video + +A dataset can hold audio and video recordings next to its time-series, to view and label them in the web app. A recording is placed on the dataset's timeline by `start`, the unix ms at which it begins, so it lines up with the sensor data. Recordings are not used for training. + +```js +// add a recording to an existing dataset +const media = await edgeML.uploadMedia("backendUrl", "deviceApiKey", datasetId, file, { + start: 1618760114000, // optional: defaults to the start of the dataset + name: "front camera", // optional: defaults to the file name + kind: "video", // optional: "audio" | "video", detected from the file +}); +// media: { _id, kind, name, start, end, durationMs, mimeType, size, meta: { width, height, fps, ... } } +``` + +`file` is a `File` or `Blob` in the browser (from an `` or a `MediaRecorder`), and a `Buffer`, `ArrayBuffer` or file path in Node.js. Pass `fileName` for a Buffer if you want the recording named after it. + +```js +// or with the dataset; a dataset with only recordings needs no timeSeries +const datasetId = await edgeML.sendDataset("backendUrl", "deviceApiKey", { + name: "walk-1", + timeSeries: [{ name: "accX", data: [[1618760114000, 1.23]] }], + media: [{ file: "./walk-1.mp4", start: 1618760114000 }], +}); + +// list datasets with their time-series and recordings (read key), and download one +const datasets = await edgeML.listDatasets("backendUrl", "readKey"); +const bytes = await edgeML.downloadMedia("backendUrl", "readKey", datasetId, datasets[0].mediaSeries[0]._id); +``` + +The server stores the file as it is and accepts what browsers can play: MP4 (H.264 with AAC) and WebM video; MP3, WAV, OGG, FLAC and M4A audio, up to 2 GB. Anything else is refused with a `400` that says why. + ## Train models ```js @@ -220,6 +251,12 @@ catalog.quantities.map((q) => [q.id, q.defaultUnit, q.units.map((u) => u.id)]); // [["acceleration", "m/s^2", ["m/s^2", "g", "mg", "cm/s^2"]], ["angular_velocity", "deg/s", ["deg/s", "rad/s", "rpm"]], ...] ``` +## Unreleased: audio and video + +- `uploadMedia` adds an audio or video recording to a dataset; `sendDataset` takes them as `media`, also without any time-series. +- `listDatasets` lists the project's datasets with their time-series and recordings; `downloadMedia` fetches a recording. +- These need an edge-ml server with media support. + ## 4.1.0: units - Time-series can carry a unit and quantity when uploading (`{ name, unit, quantity }` instead of a name). diff --git a/__tests__/test_exports.js b/__tests__/test_exports.js index b0e49b7..b7bd185 100644 --- a/__tests__/test_exports.js +++ b/__tests__/test_exports.js @@ -7,7 +7,7 @@ const edgeML = require("../dist/index"); describe("Package exports", () => { it("exposes the documented API", () => { for (const name of [ - "datasetCollector", "sendDataset", "fetchUnits", "UnitError", + "datasetCollector", "sendDataset", "listDatasets", "uploadMedia", "downloadMedia", "fetchUnits", "UnitError", "listPipelines", "trainModel", "startTraining", "waitForModel", "getModel", "listModels", "loadModel", "OnnxPredictor", "PredictorError", ]) { diff --git a/__tests__/test_media.js b/__tests__/test_media.js new file mode 100644 index 0000000..8e35422 --- /dev/null +++ b/__tests__/test_media.js @@ -0,0 +1,161 @@ +const fs = require("fs"); +const os = require("os"); +const path = require("path"); +const edgeML = require("../dist/index"); + +const URL = "https://edge-ml.example/"; +const MEDIA = { _id: "m1", kind: "audio", name: "voice", start: 1000, end: 3000, durationMs: 2000 }; + +// Replaces global fetch with a router; records every call. Multipart bodies +// are recorded as { field: value }, files as { name, bytes }. +function mockFetch(routes) { + const calls = []; + global.fetch = jest.fn(async (url, init = {}) => { + let body; + if (init.body instanceof FormData) { + body = {}; + for (const [field, value] of init.body.entries()) { + body[field] = typeof value === "string" + ? value + : { name: value.name, bytes: Buffer.from(await value.arrayBuffer()).toString() }; + } + } else if (init.body) { + body = JSON.parse(init.body); + } + const call = { url, method: init.method, headers: init.headers, body }; + calls.push(call); + const key = `${init.method} ${url.replace("https://edge-ml.example", "")}`; + const handler = routes[key]; + if (!handler) return new Response(JSON.stringify({ detail: "no route " + key }), { status: 404 }); + const res = await handler(body, calls); + if (res instanceof Response) return res; + return new Response(JSON.stringify(res), { status: 200, headers: { "Content-Type": "application/json" } }); + }); + return calls; +} + +afterEach(() => { + delete global.fetch; +}); + +describe("uploadMedia", () => { + it("sends a Buffer as multipart form data", async () => { + const calls = mockFetch({ "POST /api/v1/deviceapi/dataset/media/WRITE/d1": () => MEDIA }); + const media = await edgeML.uploadMedia(URL, "WRITE", "d1", Buffer.from("sound"), { + kind: "audio", name: "voice", start: 1000.4, fileName: "voice.mp3", + }); + expect(media).toEqual(MEDIA); + expect(calls[0].body).toEqual({ + file: { name: "voice.mp3", bytes: "sound" }, kind: "audio", name: "voice", start: "1000", + }); + // fetch has to set the multipart boundary itself + expect(calls[0].headers).toBeUndefined(); + }); + + it("sends only what was given", async () => { + const calls = mockFetch({ "POST /api/v1/deviceapi/dataset/media/WRITE/d1": () => MEDIA }); + await edgeML.uploadMedia(URL, "WRITE", "d1", new Uint8Array([104, 105]).buffer); + expect(calls[0].body).toEqual({ file: { name: "recording", bytes: "hi" } }); + }); + + it("takes a Blob or File", async () => { + const calls = mockFetch({ "POST /api/v1/deviceapi/dataset/media/WRITE/d1": () => MEDIA }); + await edgeML.uploadMedia(URL, "WRITE", "d1", new File(["clip"], "clip.webm", { type: "video/webm" })); + expect(calls[0].body.file).toEqual({ name: "clip.webm", bytes: "clip" }); + }); + + it("reads a file path in Node.js", async () => { + const file = path.join(fs.mkdtempSync(path.join(os.tmpdir(), "edgeml-")), "walk.mp4"); + fs.writeFileSync(file, "frames"); + const calls = mockFetch({ "POST /api/v1/deviceapi/dataset/media/WRITE/d1": () => MEDIA }); + await edgeML.uploadMedia(URL, "WRITE", "d1", file); + expect(calls[0].body.file).toEqual({ name: "walk.mp4", bytes: "frames" }); + await expect(edgeML.uploadMedia(URL, "WRITE", "d1", file + ".missing")).rejects.toThrow("ENOENT"); + }); + + it("rejects bad arguments before sending anything", async () => { + const calls = mockFetch({}); + const buf = Buffer.from("x"); + await expect(edgeML.uploadMedia(URL, "WRITE", "", buf)).rejects.toThrow("needs the id of a dataset"); + await expect(edgeML.uploadMedia(URL, "WRITE", "d1", buf, { kind: "photo" })).rejects.toThrow("'audio' or 'video'"); + await expect(edgeML.uploadMedia(URL, "WRITE", "d1", buf, { start: "now" })).rejects.toThrow("unix timestamp"); + await expect(edgeML.uploadMedia(URL, "WRITE", "d1", { not: "a file" })).rejects.toThrow("file must be"); + expect(calls).toHaveLength(0); + }); + + it("passes on why the server refused a file", async () => { + mockFetch({ + "POST /api/v1/deviceapi/dataset/media/WRITE/d1": () => + new Response(JSON.stringify({ detail: "This video format (avi/mpeg4) cannot be played in the browser." }), { status: 400 }), + }); + await expect(edgeML.uploadMedia(URL, "WRITE", "d1", Buffer.from("x"))).rejects.toThrow( + "400: This video format (avi/mpeg4) cannot be played in the browser." + ); + }); +}); + +describe("sendDataset with media", () => { + it("uploads recordings after the time-series", async () => { + const calls = mockFetch({ + "POST /api/v1/deviceapi/dataset/init/WRITE": () => ({ id: "d1" }), + "POST /api/v1/deviceapi/dataset/append/WRITE/d1": () => undefined, + "POST /api/v1/deviceapi/dataset/media/WRITE/d1": () => MEDIA, + }); + const id = await edgeML.sendDataset(URL, "WRITE", { + name: "walk", + timeSeries: [{ name: "accX", data: [[1000, 1]] }], + media: [ + { file: Buffer.from("a"), fileName: "cam.mp4", start: 1000 }, + { file: Buffer.from("b"), fileName: "mic.wav", name: "microphone" }, + ], + }); + expect(id).toBe("d1"); + expect(calls.map((c) => c.url.split("/deviceapi/")[1])).toEqual([ + "dataset/init/WRITE", "dataset/append/WRITE/d1", "dataset/media/WRITE/d1", "dataset/media/WRITE/d1", + ]); + expect(calls[2].body).toEqual({ file: { name: "cam.mp4", bytes: "a" }, start: "1000" }); + expect(calls[3].body).toEqual({ file: { name: "mic.wav", bytes: "b" }, name: "microphone" }); + }); + + it("creates a dataset that only holds recordings", async () => { + const calls = mockFetch({ + "POST /api/v1/deviceapi/dataset/init/WRITE": () => ({ id: "d1" }), + "POST /api/v1/deviceapi/dataset/media/WRITE/d1": () => MEDIA, + }); + await edgeML.sendDataset(URL, "WRITE", { name: "interview", media: [{ file: Buffer.from("a") }] }); + expect(calls[0].body).toEqual({ name: "interview", metaData: {}, timeSeries: [] }); + expect(calls).toHaveLength(2); // nothing to append + }); + + it("still needs time-series or media", async () => { + mockFetch({}); + await expect(edgeML.sendDataset(URL, "WRITE", { name: "empty" })).rejects.toThrow("needs a name and a timeSeries array"); + await expect(edgeML.sendDataset(URL, "WRITE", { name: "x", timeSeries: [], media: [{}] })).rejects.toThrow("media must be"); + }); +}); + +describe("listDatasets and downloadMedia", () => { + it("lists datasets with their recordings", async () => { + mockFetch({ + "GET /api/v1/deviceapi/project/READ": () => ({ + datasets: [ + { _id: "d1", name: "walk", timeSeries: [{ name: "accX" }], mediaSeries: [MEDIA] }, + { _id: "d0", name: "from before recordings", timeSeries: [] }, + ], + labelings: [], + }), + }); + const datasets = await edgeML.listDatasets(URL, "READ"); + expect(datasets.map((d) => [d.name, d.mediaSeries.length])).toEqual([["walk", 1], ["from before recordings", 0]]); + expect(datasets[0].mediaSeries[0].durationMs).toBe(2000); + }); + + it("downloads a recording", async () => { + mockFetch({ + "GET /api/v1/deviceapi/dataset/media/READ/d1/m1": () => new Response(Buffer.from("sound"), { status: 200 }), + }); + const data = await edgeML.downloadMedia(URL, "READ", "d1", "m1"); + expect(data).toBeInstanceOf(ArrayBuffer); + expect(Buffer.from(data).toString()).toBe("sound"); + }); +}); diff --git a/dist/index.browser.esm.js b/dist/index.browser.esm.js index 3bac5f3..2dc7b3c 100644 --- a/dist/index.browser.esm.js +++ b/dist/index.browser.esm.js @@ -6,13 +6,14 @@ const joinUrl = (base, path) => base.replace(/\/+$/, "") + path; -async function request(method, base, path, body, { responseType = "json" } = {}) { +async function request(method, base, path, body, { responseType = "json", form = false } = {}) { let res; try { res = await fetch(joinUrl(base, path), { method, - headers: body !== undefined ? { "Content-Type": "application/json" } : undefined, - body: body !== undefined ? JSON.stringify(body) : undefined, + // a form is sent as multipart; fetch adds the header with its boundary + headers: body !== undefined && !form ? { "Content-Type": "application/json" } : undefined, + body: body === undefined ? undefined : form ? body : JSON.stringify(body), }); } catch (e) { throw new Error("Server error: " + (e && e.message ? e.message : e)); @@ -37,6 +38,7 @@ async function request(method, base, path, body, { responseType = "json" } = {}) const http = { get: (base, path, opts) => request("GET", base, path, undefined, opts), post: (base, path, body, opts) => request("POST", base, path, body, opts), + postForm: (base, path, form, opts) => request("POST", base, path, form, { ...opts, form: true }), }; /** Thrown by OnnxPredictor.predict while not enough data is buffered for a window. */ @@ -589,6 +591,104 @@ function loadModel(url, key, modelId, opts) { return OnnxPredictor.fromServer(url, key, modelId, opts); } +// Audio and video recordings in datasets, and listing datasets. +// +// A recording is stored as uploaded and placed on the dataset's timeline by +// its `start` (unix ms), next to the time-series. The server accepts what +// browsers can play: MP4 (H.264/AAC) and WebM video; MP3, WAV, OGG, FLAC and +// M4A audio. + +const URLS$1 = { + media: "/api/v1/deviceapi/dataset/media/", + project: "/api/v1/deviceapi/project/", +}; + +const isBlob = (value) => typeof Blob !== "undefined" && value instanceof Blob; + +const baseName = (path) => path.split(/[\\/]/).pop(); + +/** Turns what the caller passed as `file` into a Blob and a file name. */ +async function toUpload(file, fileName) { + if (typeof file === "string") { + let bytes; + try { + bytes = await readFile(file); + } catch (e) { + if (/only available in Node\.js/.test(e.message)) { + throw new Error("File paths can only be uploaded from Node.js; pass a File or Blob in the browser"); + } + throw e; + } + return { blob: new Blob([bytes]), name: fileName || baseName(file) }; + } + if (isBlob(file)) { + return { blob: file, name: fileName || file.name || "recording" }; + } + if (file instanceof ArrayBuffer || ArrayBuffer.isView(file)) { + return { blob: new Blob([file]), name: fileName || "recording" }; + } + throw new Error("file must be a File, Blob, Buffer, ArrayBuffer or (in Node.js) a file path"); +} + +/** + * Adds an audio or video recording to a dataset. + * @param {string} url - The url of the backend server + * @param {string} key - The Device-Api-Key (write access) + * @param {string} datasetId - id of the dataset, e.g. from sendDataset + * @param {Blob | ArrayBuffer | ArrayBufferView | string} file - a File or Blob + * (browser), a Buffer or ArrayBuffer, or a file path (Node.js) + * @param {import("../index").MediaOptions} [options] + * @returns {Promise} the stored recording + */ +async function uploadMedia(url, key, datasetId, file, options = {}) { + if (!datasetId) { + throw new Error("uploadMedia needs the id of a dataset"); + } + if (options.kind !== undefined && options.kind !== "audio" && options.kind !== "video") { + throw new Error("kind must be 'audio' or 'video'"); + } + if (options.start !== undefined && !Number.isFinite(options.start)) { + throw new Error("start must be a unix timestamp in milliseconds"); + } + const { blob, name } = await toUpload(file, options.fileName); + const form = new FormData(); + // the file name tells the server nothing about the format (it looks at the + // content), but names the recording when `name` is not given + form.append("file", blob, name); + if (options.kind) form.append("kind", options.kind); + if (options.name) form.append("name", options.name); + if (options.start !== undefined) form.append("start", String(Math.round(options.start))); + return http.postForm(url, URLS$1.media + key + "/" + datasetId, form); +} + +/** + * Downloads a recording as it was uploaded. + * @param {string} url - The url of the backend server + * @param {string} key - The Device-Api-Key (read or write access) + * @param {string} datasetId + * @param {string} mediaId - `_id` of an entry of the dataset's `mediaSeries` + * @returns {Promise} + */ +function downloadMedia(url, key, datasetId, mediaId) { + return http.get(url, URLS$1.media + key + "/" + datasetId + "/" + mediaId, { responseType: "arraybuffer" }); +} + +/** + * Lists the datasets of the project, with their time-series and recordings + * (no sample values). + * @param {string} url - The url of the backend server + * @param {string} key - The Device-Api-Key (read or write access) + * @returns {Promise} + */ +async function listDatasets(url, key) { + const project = await http.get(url, URLS$1.project + key); + return ((project && project.datasets) || []).map((dataset) => ({ + ...dataset, + // datasets from before recordings existed have no such list + mediaSeries: dataset.mediaSeries || [], + })); +} + const UPLOAD_INTERVAL = 5 * 1000; const URLS = { @@ -603,17 +703,24 @@ const URLS = { * @param {import("../index").DatasetUpload} dataset * time-series data as [unix ms, value] pairs, each with an optional unit and quantity * (e.g. { name: "accX", unit: "g", quantity: "acceleration", data }); labeling as - * "{labeling}_{label}" for the whole dataset + * "{labeling}_{label}" for the whole dataset; `media` with audio/video recordings + * (e.g. { file, start }), in which case timeSeries may be empty * @returns {Promise} The id of the created dataset */ async function sendDataset(url, key, dataset) { - if (!dataset || !dataset.name || !Array.isArray(dataset.timeSeries)) { + const media = dataset && dataset.media !== undefined ? dataset.media : []; + if (!Array.isArray(media) || media.some((m) => !m || m.file === undefined)) { + throw new Error("media must be an array of { file, kind?, name?, start? }"); + } + // a dataset that only holds recordings needs no timeSeries + const series = dataset && dataset.timeSeries === undefined && media.length > 0 ? [] : dataset && dataset.timeSeries; + if (!dataset || !dataset.name || !Array.isArray(series)) { throw new Error("dataset needs a name and a timeSeries array"); } const init = await http.post(url, URLS.initDatasetIncrement + key, { name: dataset.name, metaData: dataset.metaData || {}, - timeSeries: dataset.timeSeries.map(initEntry), + timeSeries: series.map(initEntry), }); if (!init || !init.id) { throw new Error("Could not create dataset"); @@ -623,10 +730,17 @@ async function sendDataset(url, key, dataset) { const [labelingName, labelName] = dataset.labeling.split("_"); labeling = { labelingName, labelName }; } - await http.post(url, URLS.addDatasetIncrement + key + "/" + init.id, { - data: dataset.timeSeries.map((ts) => ({ name: ts.name, data: ts.data })), - labeling: labeling, - }); + // a label for the whole dataset spans its time-series, so there is nothing + // to send for a dataset without any + if (series.length > 0) { + await http.post(url, URLS.addDatasetIncrement + key + "/" + init.id, { + data: series.map((ts) => ({ name: ts.name, data: ts.data })), + labeling: labeling, + }); + } + for (const { file, ...options } of media) { + await uploadMedia(url, key, init.id, file, options); + } return init.id; } @@ -773,6 +887,10 @@ const edgeML = { // data collection datasetCollector: datasetCollector, sendDataset: sendDataset, + listDatasets: listDatasets, + // audio and video + uploadMedia: uploadMedia, + downloadMedia: downloadMedia, // units fetchUnits: fetchUnits, UnitError: UnitError, @@ -789,4 +907,4 @@ const edgeML = { PredictorError: PredictorError, }; -export { OnnxPredictor, PredictorError, UnitError, datasetCollector, edgeML as default, fetchUnits, getModel, listModels, listPipelines, loadModel, sendDataset, startTraining, trainModel, waitForModel }; +export { OnnxPredictor, PredictorError, UnitError, datasetCollector, edgeML as default, downloadMedia, fetchUnits, getModel, listDatasets, listModels, listPipelines, loadModel, sendDataset, startTraining, trainModel, uploadMedia, waitForModel }; diff --git a/dist/index.browser.js b/dist/index.browser.js index 841f613..26d6150 100644 --- a/dist/index.browser.js +++ b/dist/index.browser.js @@ -12,13 +12,14 @@ const joinUrl = (base, path) => base.replace(/\/+$/, "") + path; - async function request(method, base, path, body, { responseType = "json" } = {}) { + async function request(method, base, path, body, { responseType = "json", form = false } = {}) { let res; try { res = await fetch(joinUrl(base, path), { method, - headers: body !== undefined ? { "Content-Type": "application/json" } : undefined, - body: body !== undefined ? JSON.stringify(body) : undefined, + // a form is sent as multipart; fetch adds the header with its boundary + headers: body !== undefined && !form ? { "Content-Type": "application/json" } : undefined, + body: body === undefined ? undefined : form ? body : JSON.stringify(body), }); } catch (e) { throw new Error("Server error: " + (e && e.message ? e.message : e)); @@ -43,6 +44,7 @@ const http = { get: (base, path, opts) => request("GET", base, path, undefined, opts), post: (base, path, body, opts) => request("POST", base, path, body, opts), + postForm: (base, path, form, opts) => request("POST", base, path, form, { ...opts, form: true }), }; /** Thrown by OnnxPredictor.predict while not enough data is buffered for a window. */ @@ -595,6 +597,104 @@ return OnnxPredictor.fromServer(url, key, modelId, opts); } + // Audio and video recordings in datasets, and listing datasets. + // + // A recording is stored as uploaded and placed on the dataset's timeline by + // its `start` (unix ms), next to the time-series. The server accepts what + // browsers can play: MP4 (H.264/AAC) and WebM video; MP3, WAV, OGG, FLAC and + // M4A audio. + + const URLS$1 = { + media: "/api/v1/deviceapi/dataset/media/", + project: "/api/v1/deviceapi/project/", + }; + + const isBlob = (value) => typeof Blob !== "undefined" && value instanceof Blob; + + const baseName = (path) => path.split(/[\\/]/).pop(); + + /** Turns what the caller passed as `file` into a Blob and a file name. */ + async function toUpload(file, fileName) { + if (typeof file === "string") { + let bytes; + try { + bytes = await readFile(file); + } catch (e) { + if (/only available in Node\.js/.test(e.message)) { + throw new Error("File paths can only be uploaded from Node.js; pass a File or Blob in the browser"); + } + throw e; + } + return { blob: new Blob([bytes]), name: fileName || baseName(file) }; + } + if (isBlob(file)) { + return { blob: file, name: fileName || file.name || "recording" }; + } + if (file instanceof ArrayBuffer || ArrayBuffer.isView(file)) { + return { blob: new Blob([file]), name: fileName || "recording" }; + } + throw new Error("file must be a File, Blob, Buffer, ArrayBuffer or (in Node.js) a file path"); + } + + /** + * Adds an audio or video recording to a dataset. + * @param {string} url - The url of the backend server + * @param {string} key - The Device-Api-Key (write access) + * @param {string} datasetId - id of the dataset, e.g. from sendDataset + * @param {Blob | ArrayBuffer | ArrayBufferView | string} file - a File or Blob + * (browser), a Buffer or ArrayBuffer, or a file path (Node.js) + * @param {import("../index").MediaOptions} [options] + * @returns {Promise} the stored recording + */ + async function uploadMedia(url, key, datasetId, file, options = {}) { + if (!datasetId) { + throw new Error("uploadMedia needs the id of a dataset"); + } + if (options.kind !== undefined && options.kind !== "audio" && options.kind !== "video") { + throw new Error("kind must be 'audio' or 'video'"); + } + if (options.start !== undefined && !Number.isFinite(options.start)) { + throw new Error("start must be a unix timestamp in milliseconds"); + } + const { blob, name } = await toUpload(file, options.fileName); + const form = new FormData(); + // the file name tells the server nothing about the format (it looks at the + // content), but names the recording when `name` is not given + form.append("file", blob, name); + if (options.kind) form.append("kind", options.kind); + if (options.name) form.append("name", options.name); + if (options.start !== undefined) form.append("start", String(Math.round(options.start))); + return http.postForm(url, URLS$1.media + key + "/" + datasetId, form); + } + + /** + * Downloads a recording as it was uploaded. + * @param {string} url - The url of the backend server + * @param {string} key - The Device-Api-Key (read or write access) + * @param {string} datasetId + * @param {string} mediaId - `_id` of an entry of the dataset's `mediaSeries` + * @returns {Promise} + */ + function downloadMedia(url, key, datasetId, mediaId) { + return http.get(url, URLS$1.media + key + "/" + datasetId + "/" + mediaId, { responseType: "arraybuffer" }); + } + + /** + * Lists the datasets of the project, with their time-series and recordings + * (no sample values). + * @param {string} url - The url of the backend server + * @param {string} key - The Device-Api-Key (read or write access) + * @returns {Promise} + */ + async function listDatasets(url, key) { + const project = await http.get(url, URLS$1.project + key); + return ((project && project.datasets) || []).map((dataset) => ({ + ...dataset, + // datasets from before recordings existed have no such list + mediaSeries: dataset.mediaSeries || [], + })); + } + const UPLOAD_INTERVAL = 5 * 1000; const URLS = { @@ -609,17 +709,24 @@ * @param {import("../index").DatasetUpload} dataset * time-series data as [unix ms, value] pairs, each with an optional unit and quantity * (e.g. { name: "accX", unit: "g", quantity: "acceleration", data }); labeling as - * "{labeling}_{label}" for the whole dataset + * "{labeling}_{label}" for the whole dataset; `media` with audio/video recordings + * (e.g. { file, start }), in which case timeSeries may be empty * @returns {Promise} The id of the created dataset */ async function sendDataset(url, key, dataset) { - if (!dataset || !dataset.name || !Array.isArray(dataset.timeSeries)) { + const media = dataset && dataset.media !== undefined ? dataset.media : []; + if (!Array.isArray(media) || media.some((m) => !m || m.file === undefined)) { + throw new Error("media must be an array of { file, kind?, name?, start? }"); + } + // a dataset that only holds recordings needs no timeSeries + const series = dataset && dataset.timeSeries === undefined && media.length > 0 ? [] : dataset && dataset.timeSeries; + if (!dataset || !dataset.name || !Array.isArray(series)) { throw new Error("dataset needs a name and a timeSeries array"); } const init = await http.post(url, URLS.initDatasetIncrement + key, { name: dataset.name, metaData: dataset.metaData || {}, - timeSeries: dataset.timeSeries.map(initEntry), + timeSeries: series.map(initEntry), }); if (!init || !init.id) { throw new Error("Could not create dataset"); @@ -629,10 +736,17 @@ const [labelingName, labelName] = dataset.labeling.split("_"); labeling = { labelingName, labelName }; } - await http.post(url, URLS.addDatasetIncrement + key + "/" + init.id, { - data: dataset.timeSeries.map((ts) => ({ name: ts.name, data: ts.data })), - labeling: labeling, - }); + // a label for the whole dataset spans its time-series, so there is nothing + // to send for a dataset without any + if (series.length > 0) { + await http.post(url, URLS.addDatasetIncrement + key + "/" + init.id, { + data: series.map((ts) => ({ name: ts.name, data: ts.data })), + labeling: labeling, + }); + } + for (const { file, ...options } of media) { + await uploadMedia(url, key, init.id, file, options); + } return init.id; } @@ -779,6 +893,10 @@ // data collection datasetCollector: datasetCollector, sendDataset: sendDataset, + listDatasets: listDatasets, + // audio and video + uploadMedia: uploadMedia, + downloadMedia: downloadMedia, // units fetchUnits: fetchUnits, UnitError: UnitError, diff --git a/dist/index.js b/dist/index.js index 7ce091f..d3455b9 100644 --- a/dist/index.js +++ b/dist/index.js @@ -25,13 +25,14 @@ function _interopNamespaceDefault(e) { const joinUrl = (base, path) => base.replace(/\/+$/, "") + path; -async function request(method, base, path, body, { responseType = "json" } = {}) { +async function request(method, base, path, body, { responseType = "json", form = false } = {}) { let res; try { res = await fetch(joinUrl(base, path), { method, - headers: body !== undefined ? { "Content-Type": "application/json" } : undefined, - body: body !== undefined ? JSON.stringify(body) : undefined, + // a form is sent as multipart; fetch adds the header with its boundary + headers: body !== undefined && !form ? { "Content-Type": "application/json" } : undefined, + body: body === undefined ? undefined : form ? body : JSON.stringify(body), }); } catch (e) { throw new Error("Server error: " + (e && e.message ? e.message : e)); @@ -56,6 +57,7 @@ async function request(method, base, path, body, { responseType = "json" } = {}) const http = { get: (base, path, opts) => request("GET", base, path, undefined, opts), post: (base, path, body, opts) => request("POST", base, path, body, opts), + postForm: (base, path, form, opts) => request("POST", base, path, form, { ...opts, form: true }), }; /** Thrown by OnnxPredictor.predict while not enough data is buffered for a window. */ @@ -613,6 +615,104 @@ function loadModel(url, key, modelId, opts) { return OnnxPredictor.fromServer(url, key, modelId, opts); } +// Audio and video recordings in datasets, and listing datasets. +// +// A recording is stored as uploaded and placed on the dataset's timeline by +// its `start` (unix ms), next to the time-series. The server accepts what +// browsers can play: MP4 (H.264/AAC) and WebM video; MP3, WAV, OGG, FLAC and +// M4A audio. + +const URLS$1 = { + media: "/api/v1/deviceapi/dataset/media/", + project: "/api/v1/deviceapi/project/", +}; + +const isBlob = (value) => typeof Blob !== "undefined" && value instanceof Blob; + +const baseName = (path) => path.split(/[\\/]/).pop(); + +/** Turns what the caller passed as `file` into a Blob and a file name. */ +async function toUpload(file, fileName) { + if (typeof file === "string") { + let bytes; + try { + bytes = await readFile(file); + } catch (e) { + if (/only available in Node\.js/.test(e.message)) { + throw new Error("File paths can only be uploaded from Node.js; pass a File or Blob in the browser"); + } + throw e; + } + return { blob: new Blob([bytes]), name: fileName || baseName(file) }; + } + if (isBlob(file)) { + return { blob: file, name: fileName || file.name || "recording" }; + } + if (file instanceof ArrayBuffer || ArrayBuffer.isView(file)) { + return { blob: new Blob([file]), name: fileName || "recording" }; + } + throw new Error("file must be a File, Blob, Buffer, ArrayBuffer or (in Node.js) a file path"); +} + +/** + * Adds an audio or video recording to a dataset. + * @param {string} url - The url of the backend server + * @param {string} key - The Device-Api-Key (write access) + * @param {string} datasetId - id of the dataset, e.g. from sendDataset + * @param {Blob | ArrayBuffer | ArrayBufferView | string} file - a File or Blob + * (browser), a Buffer or ArrayBuffer, or a file path (Node.js) + * @param {import("../index").MediaOptions} [options] + * @returns {Promise} the stored recording + */ +async function uploadMedia(url, key, datasetId, file, options = {}) { + if (!datasetId) { + throw new Error("uploadMedia needs the id of a dataset"); + } + if (options.kind !== undefined && options.kind !== "audio" && options.kind !== "video") { + throw new Error("kind must be 'audio' or 'video'"); + } + if (options.start !== undefined && !Number.isFinite(options.start)) { + throw new Error("start must be a unix timestamp in milliseconds"); + } + const { blob, name } = await toUpload(file, options.fileName); + const form = new FormData(); + // the file name tells the server nothing about the format (it looks at the + // content), but names the recording when `name` is not given + form.append("file", blob, name); + if (options.kind) form.append("kind", options.kind); + if (options.name) form.append("name", options.name); + if (options.start !== undefined) form.append("start", String(Math.round(options.start))); + return http.postForm(url, URLS$1.media + key + "/" + datasetId, form); +} + +/** + * Downloads a recording as it was uploaded. + * @param {string} url - The url of the backend server + * @param {string} key - The Device-Api-Key (read or write access) + * @param {string} datasetId + * @param {string} mediaId - `_id` of an entry of the dataset's `mediaSeries` + * @returns {Promise} + */ +function downloadMedia(url, key, datasetId, mediaId) { + return http.get(url, URLS$1.media + key + "/" + datasetId + "/" + mediaId, { responseType: "arraybuffer" }); +} + +/** + * Lists the datasets of the project, with their time-series and recordings + * (no sample values). + * @param {string} url - The url of the backend server + * @param {string} key - The Device-Api-Key (read or write access) + * @returns {Promise} + */ +async function listDatasets(url, key) { + const project = await http.get(url, URLS$1.project + key); + return ((project && project.datasets) || []).map((dataset) => ({ + ...dataset, + // datasets from before recordings existed have no such list + mediaSeries: dataset.mediaSeries || [], + })); +} + const UPLOAD_INTERVAL = 5 * 1000; const URLS = { @@ -627,17 +727,24 @@ const URLS = { * @param {import("../index").DatasetUpload} dataset * time-series data as [unix ms, value] pairs, each with an optional unit and quantity * (e.g. { name: "accX", unit: "g", quantity: "acceleration", data }); labeling as - * "{labeling}_{label}" for the whole dataset + * "{labeling}_{label}" for the whole dataset; `media` with audio/video recordings + * (e.g. { file, start }), in which case timeSeries may be empty * @returns {Promise} The id of the created dataset */ async function sendDataset(url, key, dataset) { - if (!dataset || !dataset.name || !Array.isArray(dataset.timeSeries)) { + const media = dataset && dataset.media !== undefined ? dataset.media : []; + if (!Array.isArray(media) || media.some((m) => !m || m.file === undefined)) { + throw new Error("media must be an array of { file, kind?, name?, start? }"); + } + // a dataset that only holds recordings needs no timeSeries + const series = dataset && dataset.timeSeries === undefined && media.length > 0 ? [] : dataset && dataset.timeSeries; + if (!dataset || !dataset.name || !Array.isArray(series)) { throw new Error("dataset needs a name and a timeSeries array"); } const init = await http.post(url, URLS.initDatasetIncrement + key, { name: dataset.name, metaData: dataset.metaData || {}, - timeSeries: dataset.timeSeries.map(initEntry), + timeSeries: series.map(initEntry), }); if (!init || !init.id) { throw new Error("Could not create dataset"); @@ -647,10 +754,17 @@ async function sendDataset(url, key, dataset) { const [labelingName, labelName] = dataset.labeling.split("_"); labeling = { labelingName, labelName }; } - await http.post(url, URLS.addDatasetIncrement + key + "/" + init.id, { - data: dataset.timeSeries.map((ts) => ({ name: ts.name, data: ts.data })), - labeling: labeling, - }); + // a label for the whole dataset spans its time-series, so there is nothing + // to send for a dataset without any + if (series.length > 0) { + await http.post(url, URLS.addDatasetIncrement + key + "/" + init.id, { + data: series.map((ts) => ({ name: ts.name, data: ts.data })), + labeling: labeling, + }); + } + for (const { file, ...options } of media) { + await uploadMedia(url, key, init.id, file, options); + } return init.id; } @@ -797,6 +911,10 @@ const edgeML = { // data collection datasetCollector: datasetCollector, sendDataset: sendDataset, + listDatasets: listDatasets, + // audio and video + uploadMedia: uploadMedia, + downloadMedia: downloadMedia, // units fetchUnits: fetchUnits, UnitError: UnitError, diff --git a/dist/index.mjs b/dist/index.mjs index 704a41c..5a31200 100644 --- a/dist/index.mjs +++ b/dist/index.mjs @@ -6,13 +6,14 @@ const joinUrl = (base, path) => base.replace(/\/+$/, "") + path; -async function request(method, base, path, body, { responseType = "json" } = {}) { +async function request(method, base, path, body, { responseType = "json", form = false } = {}) { let res; try { res = await fetch(joinUrl(base, path), { method, - headers: body !== undefined ? { "Content-Type": "application/json" } : undefined, - body: body !== undefined ? JSON.stringify(body) : undefined, + // a form is sent as multipart; fetch adds the header with its boundary + headers: body !== undefined && !form ? { "Content-Type": "application/json" } : undefined, + body: body === undefined ? undefined : form ? body : JSON.stringify(body), }); } catch (e) { throw new Error("Server error: " + (e && e.message ? e.message : e)); @@ -37,6 +38,7 @@ async function request(method, base, path, body, { responseType = "json" } = {}) const http = { get: (base, path, opts) => request("GET", base, path, undefined, opts), post: (base, path, body, opts) => request("POST", base, path, body, opts), + postForm: (base, path, form, opts) => request("POST", base, path, form, { ...opts, form: true }), }; /** Thrown by OnnxPredictor.predict while not enough data is buffered for a window. */ @@ -594,6 +596,104 @@ function loadModel(url, key, modelId, opts) { return OnnxPredictor.fromServer(url, key, modelId, opts); } +// Audio and video recordings in datasets, and listing datasets. +// +// A recording is stored as uploaded and placed on the dataset's timeline by +// its `start` (unix ms), next to the time-series. The server accepts what +// browsers can play: MP4 (H.264/AAC) and WebM video; MP3, WAV, OGG, FLAC and +// M4A audio. + +const URLS$1 = { + media: "/api/v1/deviceapi/dataset/media/", + project: "/api/v1/deviceapi/project/", +}; + +const isBlob = (value) => typeof Blob !== "undefined" && value instanceof Blob; + +const baseName = (path) => path.split(/[\\/]/).pop(); + +/** Turns what the caller passed as `file` into a Blob and a file name. */ +async function toUpload(file, fileName) { + if (typeof file === "string") { + let bytes; + try { + bytes = await readFile(file); + } catch (e) { + if (/only available in Node\.js/.test(e.message)) { + throw new Error("File paths can only be uploaded from Node.js; pass a File or Blob in the browser"); + } + throw e; + } + return { blob: new Blob([bytes]), name: fileName || baseName(file) }; + } + if (isBlob(file)) { + return { blob: file, name: fileName || file.name || "recording" }; + } + if (file instanceof ArrayBuffer || ArrayBuffer.isView(file)) { + return { blob: new Blob([file]), name: fileName || "recording" }; + } + throw new Error("file must be a File, Blob, Buffer, ArrayBuffer or (in Node.js) a file path"); +} + +/** + * Adds an audio or video recording to a dataset. + * @param {string} url - The url of the backend server + * @param {string} key - The Device-Api-Key (write access) + * @param {string} datasetId - id of the dataset, e.g. from sendDataset + * @param {Blob | ArrayBuffer | ArrayBufferView | string} file - a File or Blob + * (browser), a Buffer or ArrayBuffer, or a file path (Node.js) + * @param {import("../index").MediaOptions} [options] + * @returns {Promise} the stored recording + */ +async function uploadMedia(url, key, datasetId, file, options = {}) { + if (!datasetId) { + throw new Error("uploadMedia needs the id of a dataset"); + } + if (options.kind !== undefined && options.kind !== "audio" && options.kind !== "video") { + throw new Error("kind must be 'audio' or 'video'"); + } + if (options.start !== undefined && !Number.isFinite(options.start)) { + throw new Error("start must be a unix timestamp in milliseconds"); + } + const { blob, name } = await toUpload(file, options.fileName); + const form = new FormData(); + // the file name tells the server nothing about the format (it looks at the + // content), but names the recording when `name` is not given + form.append("file", blob, name); + if (options.kind) form.append("kind", options.kind); + if (options.name) form.append("name", options.name); + if (options.start !== undefined) form.append("start", String(Math.round(options.start))); + return http.postForm(url, URLS$1.media + key + "/" + datasetId, form); +} + +/** + * Downloads a recording as it was uploaded. + * @param {string} url - The url of the backend server + * @param {string} key - The Device-Api-Key (read or write access) + * @param {string} datasetId + * @param {string} mediaId - `_id` of an entry of the dataset's `mediaSeries` + * @returns {Promise} + */ +function downloadMedia(url, key, datasetId, mediaId) { + return http.get(url, URLS$1.media + key + "/" + datasetId + "/" + mediaId, { responseType: "arraybuffer" }); +} + +/** + * Lists the datasets of the project, with their time-series and recordings + * (no sample values). + * @param {string} url - The url of the backend server + * @param {string} key - The Device-Api-Key (read or write access) + * @returns {Promise} + */ +async function listDatasets(url, key) { + const project = await http.get(url, URLS$1.project + key); + return ((project && project.datasets) || []).map((dataset) => ({ + ...dataset, + // datasets from before recordings existed have no such list + mediaSeries: dataset.mediaSeries || [], + })); +} + const UPLOAD_INTERVAL = 5 * 1000; const URLS = { @@ -608,17 +708,24 @@ const URLS = { * @param {import("../index").DatasetUpload} dataset * time-series data as [unix ms, value] pairs, each with an optional unit and quantity * (e.g. { name: "accX", unit: "g", quantity: "acceleration", data }); labeling as - * "{labeling}_{label}" for the whole dataset + * "{labeling}_{label}" for the whole dataset; `media` with audio/video recordings + * (e.g. { file, start }), in which case timeSeries may be empty * @returns {Promise} The id of the created dataset */ async function sendDataset(url, key, dataset) { - if (!dataset || !dataset.name || !Array.isArray(dataset.timeSeries)) { + const media = dataset && dataset.media !== undefined ? dataset.media : []; + if (!Array.isArray(media) || media.some((m) => !m || m.file === undefined)) { + throw new Error("media must be an array of { file, kind?, name?, start? }"); + } + // a dataset that only holds recordings needs no timeSeries + const series = dataset && dataset.timeSeries === undefined && media.length > 0 ? [] : dataset && dataset.timeSeries; + if (!dataset || !dataset.name || !Array.isArray(series)) { throw new Error("dataset needs a name and a timeSeries array"); } const init = await http.post(url, URLS.initDatasetIncrement + key, { name: dataset.name, metaData: dataset.metaData || {}, - timeSeries: dataset.timeSeries.map(initEntry), + timeSeries: series.map(initEntry), }); if (!init || !init.id) { throw new Error("Could not create dataset"); @@ -628,10 +735,17 @@ async function sendDataset(url, key, dataset) { const [labelingName, labelName] = dataset.labeling.split("_"); labeling = { labelingName, labelName }; } - await http.post(url, URLS.addDatasetIncrement + key + "/" + init.id, { - data: dataset.timeSeries.map((ts) => ({ name: ts.name, data: ts.data })), - labeling: labeling, - }); + // a label for the whole dataset spans its time-series, so there is nothing + // to send for a dataset without any + if (series.length > 0) { + await http.post(url, URLS.addDatasetIncrement + key + "/" + init.id, { + data: series.map((ts) => ({ name: ts.name, data: ts.data })), + labeling: labeling, + }); + } + for (const { file, ...options } of media) { + await uploadMedia(url, key, init.id, file, options); + } return init.id; } @@ -778,6 +892,10 @@ const edgeML = { // data collection datasetCollector: datasetCollector, sendDataset: sendDataset, + listDatasets: listDatasets, + // audio and video + uploadMedia: uploadMedia, + downloadMedia: downloadMedia, // units fetchUnits: fetchUnits, UnitError: UnitError, @@ -794,4 +912,4 @@ const edgeML = { PredictorError: PredictorError, }; -export { OnnxPredictor, PredictorError, UnitError, datasetCollector, edgeML as default, fetchUnits, getModel, listModels, listPipelines, loadModel, sendDataset, startTraining, trainModel, waitForModel }; +export { OnnxPredictor, PredictorError, UnitError, datasetCollector, edgeML as default, downloadMedia, fetchUnits, getModel, listDatasets, listModels, listPipelines, loadModel, sendDataset, startTraining, trainModel, uploadMedia, waitForModel }; diff --git a/index.d.ts b/index.d.ts index b091716..9676c20 100644 --- a/index.d.ts +++ b/index.d.ts @@ -50,10 +50,89 @@ export interface DatasetUpload { metaData?: Record; /** label for the whole dataset as "{labeling}_{label}" */ labeling?: string; + /** audio and video recordings to add to the dataset */ + media?: ({ file: MediaFile } & MediaOptions)[]; } +/** A dataset that only holds recordings: `timeSeries` can be left out. */ +export type MediaDatasetUpload = Omit & { + timeSeries?: DatasetUpload["timeSeries"]; + media: ({ file: MediaFile } & MediaOptions)[]; +}; + /** Uploads a whole dataset. Resolves to the id of the new dataset. */ -export function sendDataset(url: string, key: string, dataset: DatasetUpload): Promise; +export function sendDataset(url: string, key: string, dataset: DatasetUpload | MediaDatasetUpload): Promise; + +// ---------------------------------------------------------------- audio and video + +/** A File or Blob (browser), a Buffer or ArrayBuffer, or a file path (Node.js). */ +export type MediaFile = Blob | ArrayBuffer | ArrayBufferView | string; + +export interface MediaOptions { + /** checked against the file; detected from it when omitted */ + kind?: "audio" | "video"; + /** name of the recording; defaults to the file name */ + name?: string; + /** + * unix ms at which the recording begins on the dataset timeline; defaults + * to the start of the dataset (or now, for a dataset without data) + */ + start?: number; + /** file name to send for a Buffer/ArrayBuffer, which has none */ + fileName?: string; +} + +/** An audio or video recording of a dataset. */ +export interface MediaSeries { + _id: string; + kind: "audio" | "video"; + name: string; + /** unix ms at which the recording begins on the dataset timeline */ + start: number; + /** start + durationMs */ + end: number; + durationMs: number; + mimeType: string; + /** bytes */ + size: number; + /** "processing" while the waveform shown in the web app is computed */ + status: "processing" | "ready"; + hasPeaks: boolean; + meta: { + container?: string; + videoCodec?: string | null; + audioCodec?: string | null; + width?: number; + height?: number; + fps?: number; + sampleRate?: number; + channels?: number; + }; +} + +/** + * Adds an audio or video recording to a dataset (write key). The server + * accepts what browsers can play: MP4 (H.264/AAC) and WebM video; MP3, WAV, + * OGG, FLAC and M4A audio. + */ +export function uploadMedia( + url: string, key: string, datasetId: string, file: MediaFile, options?: MediaOptions +): Promise; + +/** Downloads a recording as it was uploaded. */ +export function downloadMedia(url: string, key: string, datasetId: string, mediaId: string): Promise; + +export interface DatasetInfo { + _id: string; + name: string; + metaData: Record; + timeSeries: ({ _id: string; name: string; start: number | null; end: number | null; length?: number | null } & SeriesUnit)[]; + mediaSeries: MediaSeries[]; + labelings: { labelingId: string; labels: { _id: string; start: number; end: number; type: string }[] }[]; +} + +/** Lists the datasets of the project with their time-series and recordings (no sample values). */ +export function listDatasets(url: string, key: string): Promise; // ---------------------------------------------------------------- units @@ -243,6 +322,9 @@ export function loadModel(url: string, key: string, modelId: string, opts?: Load declare const edgeML: { datasetCollector: typeof datasetCollector; sendDataset: typeof sendDataset; + listDatasets: typeof listDatasets; + uploadMedia: typeof uploadMedia; + downloadMedia: typeof downloadMedia; fetchUnits: typeof fetchUnits; UnitError: typeof UnitError; listPipelines: typeof listPipelines; diff --git a/rollup.config.mjs b/rollup.config.mjs index e8605d7..4745b6b 100644 --- a/rollup.config.mjs +++ b/rollup.config.mjs @@ -9,7 +9,7 @@ const nodeExternal = ['onnxruntime-node', 'onnxruntime-web', 'fs']; // no Node-only code (onnxruntime-node, fs) in browser builds const browserPlugins = () => [ - alias({ entries: [{ find: /^\.\/platform\.node$/, replacement: src('onnx/platform.browser.js') }] }), + alias({ entries: [{ find: /^\.\/(onnx\/)?platform\.node$/, replacement: src('onnx/platform.browser.js') }] }), ]; // index.js has named exports plus a default export (ES modules); diff --git a/src/http.js b/src/http.js index 64f7b60..4ffcd0b 100644 --- a/src/http.js +++ b/src/http.js @@ -6,13 +6,14 @@ const joinUrl = (base, path) => base.replace(/\/+$/, "") + path; -async function request(method, base, path, body, { responseType = "json" } = {}) { +async function request(method, base, path, body, { responseType = "json", form = false } = {}) { let res; try { res = await fetch(joinUrl(base, path), { method, - headers: body !== undefined ? { "Content-Type": "application/json" } : undefined, - body: body !== undefined ? JSON.stringify(body) : undefined, + // a form is sent as multipart; fetch adds the header with its boundary + headers: body !== undefined && !form ? { "Content-Type": "application/json" } : undefined, + body: body === undefined ? undefined : form ? body : JSON.stringify(body), }); } catch (e) { throw new Error("Server error: " + (e && e.message ? e.message : e)); @@ -37,4 +38,5 @@ async function request(method, base, path, body, { responseType = "json" } = {}) export const http = { get: (base, path, opts) => request("GET", base, path, undefined, opts), post: (base, path, body, opts) => request("POST", base, path, body, opts), + postForm: (base, path, form, opts) => request("POST", base, path, form, { ...opts, form: true }), }; diff --git a/src/index.js b/src/index.js index 738bc02..2064b20 100644 --- a/src/index.js +++ b/src/index.js @@ -2,6 +2,7 @@ import { http } from "./http"; import { PredictorError } from "./errors"; import { OnnxPredictor, loadModel } from "./onnx/OnnxPredictor"; import { UnitError, fetchUnits, initEntry, seriesSpec } from "./units"; +import { uploadMedia, downloadMedia, listDatasets } from "./media"; import { listPipelines, startTraining, trainModel, waitForModel, getModel, listModels } from "./training"; const UPLOAD_INTERVAL = 5 * 1000; @@ -18,17 +19,24 @@ const URLS = { * @param {import("../index").DatasetUpload} dataset * time-series data as [unix ms, value] pairs, each with an optional unit and quantity * (e.g. { name: "accX", unit: "g", quantity: "acceleration", data }); labeling as - * "{labeling}_{label}" for the whole dataset + * "{labeling}_{label}" for the whole dataset; `media` with audio/video recordings + * (e.g. { file, start }), in which case timeSeries may be empty * @returns {Promise} The id of the created dataset */ async function sendDataset(url, key, dataset) { - if (!dataset || !dataset.name || !Array.isArray(dataset.timeSeries)) { + const media = dataset && dataset.media !== undefined ? dataset.media : []; + if (!Array.isArray(media) || media.some((m) => !m || m.file === undefined)) { + throw new Error("media must be an array of { file, kind?, name?, start? }"); + } + // a dataset that only holds recordings needs no timeSeries + const series = dataset && dataset.timeSeries === undefined && media.length > 0 ? [] : dataset && dataset.timeSeries; + if (!dataset || !dataset.name || !Array.isArray(series)) { throw new Error("dataset needs a name and a timeSeries array"); } const init = await http.post(url, URLS.initDatasetIncrement + key, { name: dataset.name, metaData: dataset.metaData || {}, - timeSeries: dataset.timeSeries.map(initEntry), + timeSeries: series.map(initEntry), }); if (!init || !init.id) { throw new Error("Could not create dataset"); @@ -38,10 +46,17 @@ async function sendDataset(url, key, dataset) { const [labelingName, labelName] = dataset.labeling.split("_"); labeling = { labelingName, labelName }; } - await http.post(url, URLS.addDatasetIncrement + key + "/" + init.id, { - data: dataset.timeSeries.map((ts) => ({ name: ts.name, data: ts.data })), - labeling: labeling, - }); + // a label for the whole dataset spans its time-series, so there is nothing + // to send for a dataset without any + if (series.length > 0) { + await http.post(url, URLS.addDatasetIncrement + key + "/" + init.id, { + data: series.map((ts) => ({ name: ts.name, data: ts.data })), + labeling: labeling, + }); + } + for (const { file, ...options } of media) { + await uploadMedia(url, key, init.id, file, options); + } return init.id; } @@ -188,6 +203,10 @@ const edgeML = { // data collection datasetCollector: datasetCollector, sendDataset: sendDataset, + listDatasets: listDatasets, + // audio and video + uploadMedia: uploadMedia, + downloadMedia: downloadMedia, // units fetchUnits: fetchUnits, UnitError: UnitError, @@ -208,6 +227,9 @@ export default edgeML; export { datasetCollector, sendDataset, + listDatasets, + uploadMedia, + downloadMedia, fetchUnits, UnitError, listPipelines, diff --git a/src/media.js b/src/media.js new file mode 100644 index 0000000..206f18f --- /dev/null +++ b/src/media.js @@ -0,0 +1,101 @@ +// Audio and video recordings in datasets, and listing datasets. +// +// A recording is stored as uploaded and placed on the dataset's timeline by +// its `start` (unix ms), next to the time-series. The server accepts what +// browsers can play: MP4 (H.264/AAC) and WebM video; MP3, WAV, OGG, FLAC and +// M4A audio. +import { http } from "./http"; +import { readFile } from "./onnx/platform.node"; + +const URLS = { + media: "/api/v1/deviceapi/dataset/media/", + project: "/api/v1/deviceapi/project/", +}; + +const isBlob = (value) => typeof Blob !== "undefined" && value instanceof Blob; + +const baseName = (path) => path.split(/[\\/]/).pop(); + +/** Turns what the caller passed as `file` into a Blob and a file name. */ +async function toUpload(file, fileName) { + if (typeof file === "string") { + let bytes; + try { + bytes = await readFile(file); + } catch (e) { + if (/only available in Node\.js/.test(e.message)) { + throw new Error("File paths can only be uploaded from Node.js; pass a File or Blob in the browser"); + } + throw e; + } + return { blob: new Blob([bytes]), name: fileName || baseName(file) }; + } + if (isBlob(file)) { + return { blob: file, name: fileName || file.name || "recording" }; + } + if (file instanceof ArrayBuffer || ArrayBuffer.isView(file)) { + return { blob: new Blob([file]), name: fileName || "recording" }; + } + throw new Error("file must be a File, Blob, Buffer, ArrayBuffer or (in Node.js) a file path"); +} + +/** + * Adds an audio or video recording to a dataset. + * @param {string} url - The url of the backend server + * @param {string} key - The Device-Api-Key (write access) + * @param {string} datasetId - id of the dataset, e.g. from sendDataset + * @param {Blob | ArrayBuffer | ArrayBufferView | string} file - a File or Blob + * (browser), a Buffer or ArrayBuffer, or a file path (Node.js) + * @param {import("../index").MediaOptions} [options] + * @returns {Promise} the stored recording + */ +async function uploadMedia(url, key, datasetId, file, options = {}) { + if (!datasetId) { + throw new Error("uploadMedia needs the id of a dataset"); + } + if (options.kind !== undefined && options.kind !== "audio" && options.kind !== "video") { + throw new Error("kind must be 'audio' or 'video'"); + } + if (options.start !== undefined && !Number.isFinite(options.start)) { + throw new Error("start must be a unix timestamp in milliseconds"); + } + const { blob, name } = await toUpload(file, options.fileName); + const form = new FormData(); + // the file name tells the server nothing about the format (it looks at the + // content), but names the recording when `name` is not given + form.append("file", blob, name); + if (options.kind) form.append("kind", options.kind); + if (options.name) form.append("name", options.name); + if (options.start !== undefined) form.append("start", String(Math.round(options.start))); + return http.postForm(url, URLS.media + key + "/" + datasetId, form); +} + +/** + * Downloads a recording as it was uploaded. + * @param {string} url - The url of the backend server + * @param {string} key - The Device-Api-Key (read or write access) + * @param {string} datasetId + * @param {string} mediaId - `_id` of an entry of the dataset's `mediaSeries` + * @returns {Promise} + */ +function downloadMedia(url, key, datasetId, mediaId) { + return http.get(url, URLS.media + key + "/" + datasetId + "/" + mediaId, { responseType: "arraybuffer" }); +} + +/** + * Lists the datasets of the project, with their time-series and recordings + * (no sample values). + * @param {string} url - The url of the backend server + * @param {string} key - The Device-Api-Key (read or write access) + * @returns {Promise} + */ +async function listDatasets(url, key) { + const project = await http.get(url, URLS.project + key); + return ((project && project.datasets) || []).map((dataset) => ({ + ...dataset, + // datasets from before recordings existed have no such list + mediaSeries: dataset.mediaSeries || [], + })); +} + +export { uploadMedia, downloadMedia, listDatasets };