"use strict";
|
/**
|
* @license
|
* Copyright 2018 Google LLC. All Rights Reserved.
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
* you may not use this file except in compliance with the License.
|
* You may obtain a copy of the License at
|
*
|
* http://www.apache.org/licenses/LICENSE-2.0
|
*
|
* Unless required by applicable law or agreed to in writing, software
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
* See the License for the specific language governing permissions and
|
* limitations under the License.
|
* =============================================================================
|
*/
|
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
return new (P || (P = Promise))(function (resolve, reject) {
|
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
function step(result) { result.done ? resolve(result.value) : new P(function (resolve) { resolve(result.value); }).then(fulfilled, rejected); }
|
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
});
|
};
|
var __generator = (this && this.__generator) || function (thisArg, body) {
|
var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;
|
return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
|
function verb(n) { return function (v) { return step([n, v]); }; }
|
function step(op) {
|
if (f) throw new TypeError("Generator is already executing.");
|
while (_) try {
|
if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
|
if (y = 0, t) op = [op[0] & 2, t.value];
|
switch (op[0]) {
|
case 0: case 1: t = op; break;
|
case 4: _.label++; return { value: op[1], done: false };
|
case 5: _.label++; y = op[1]; op = [0]; continue;
|
case 7: op = _.ops.pop(); _.trys.pop(); continue;
|
default:
|
if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
|
if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
|
if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
|
if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
|
if (t[2]) _.ops.pop();
|
_.trys.pop(); continue;
|
}
|
op = body.call(thisArg, _);
|
} catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
|
if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
|
}
|
};
|
Object.defineProperty(exports, "__esModule", { value: true });
|
var environment_1 = require("../environment");
|
var util_1 = require("../util");
|
var io_utils_1 = require("./io_utils");
|
var model_management_1 = require("./model_management");
|
var router_registry_1 = require("./router_registry");
|
var PATH_SEPARATOR = '/';
|
var PATH_PREFIX = 'tensorflowjs_models';
|
var INFO_SUFFIX = 'info';
|
var MODEL_TOPOLOGY_SUFFIX = 'model_topology';
|
var WEIGHT_SPECS_SUFFIX = 'weight_specs';
|
var WEIGHT_DATA_SUFFIX = 'weight_data';
|
var MODEL_METADATA_SUFFIX = 'model_metadata';
|
/**
|
* Purge all tensorflow.js-saved model artifacts from local storage.
|
*
|
* @returns Paths of the models purged.
|
*/
|
function purgeLocalStorageArtifacts() {
|
if (!environment_1.env().getBool('IS_BROWSER') ||
|
typeof window === 'undefined' ||
|
typeof window.localStorage === 'undefined') {
|
throw new Error('purgeLocalStorageModels() cannot proceed because local storage is ' +
|
'unavailable in the current environment.');
|
}
|
var LS = window.localStorage;
|
var purgedModelPaths = [];
|
for (var i = 0; i < LS.length; ++i) {
|
var key = LS.key(i);
|
var prefix = PATH_PREFIX + PATH_SEPARATOR;
|
if (key.startsWith(prefix) && key.length > prefix.length) {
|
LS.removeItem(key);
|
var modelName = getModelPathFromKey(key);
|
if (purgedModelPaths.indexOf(modelName) === -1) {
|
purgedModelPaths.push(modelName);
|
}
|
}
|
}
|
return purgedModelPaths;
|
}
|
exports.purgeLocalStorageArtifacts = purgeLocalStorageArtifacts;
|
function getModelKeys(path) {
|
return {
|
info: [PATH_PREFIX, path, INFO_SUFFIX].join(PATH_SEPARATOR),
|
topology: [PATH_PREFIX, path, MODEL_TOPOLOGY_SUFFIX].join(PATH_SEPARATOR),
|
weightSpecs: [PATH_PREFIX, path, WEIGHT_SPECS_SUFFIX].join(PATH_SEPARATOR),
|
weightData: [PATH_PREFIX, path, WEIGHT_DATA_SUFFIX].join(PATH_SEPARATOR),
|
modelMetadata: [PATH_PREFIX, path, MODEL_METADATA_SUFFIX].join(PATH_SEPARATOR)
|
};
|
}
|
/**
|
* Get model path from a local-storage key.
|
*
|
* E.g., 'tensorflowjs_models/my/model/1/info' --> 'my/model/1'
|
*
|
* @param key
|
*/
|
function getModelPathFromKey(key) {
|
var items = key.split(PATH_SEPARATOR);
|
if (items.length < 3) {
|
throw new Error("Invalid key format: " + key);
|
}
|
return items.slice(1, items.length - 1).join(PATH_SEPARATOR);
|
}
|
function maybeStripScheme(key) {
|
return key.startsWith(BrowserLocalStorage.URL_SCHEME) ?
|
key.slice(BrowserLocalStorage.URL_SCHEME.length) :
|
key;
|
}
|
/**
|
* IOHandler subclass: Browser Local Storage.
|
*
|
* See the doc string to `browserLocalStorage` for more details.
|
*/
|
var BrowserLocalStorage = /** @class */ (function () {
|
function BrowserLocalStorage(modelPath) {
|
if (!environment_1.env().getBool('IS_BROWSER') ||
|
typeof window === 'undefined' ||
|
typeof window.localStorage === 'undefined') {
|
// TODO(cais): Add more info about what IOHandler subtypes are
|
// available.
|
// Maybe point to a doc page on the web and/or automatically determine
|
// the available IOHandlers and print them in the error message.
|
throw new Error('The current environment does not support local storage.');
|
}
|
this.LS = window.localStorage;
|
if (modelPath == null || !modelPath) {
|
throw new Error('For local storage, modelPath must not be null, undefined or empty.');
|
}
|
this.modelPath = modelPath;
|
this.keys = getModelKeys(this.modelPath);
|
}
|
/**
|
* Save model artifacts to browser local storage.
|
*
|
* See the documentation to `browserLocalStorage` for details on the saved
|
* artifacts.
|
*
|
* @param modelArtifacts The model artifacts to be stored.
|
* @returns An instance of SaveResult.
|
*/
|
BrowserLocalStorage.prototype.save = function (modelArtifacts) {
|
return __awaiter(this, void 0, void 0, function () {
|
var topology, weightSpecs, modelArtifactsInfo;
|
return __generator(this, function (_a) {
|
if (modelArtifacts.modelTopology instanceof ArrayBuffer) {
|
throw new Error('BrowserLocalStorage.save() does not support saving model topology ' +
|
'in binary formats yet.');
|
}
|
else {
|
topology = JSON.stringify(modelArtifacts.modelTopology);
|
weightSpecs = JSON.stringify(modelArtifacts.weightSpecs);
|
modelArtifactsInfo = io_utils_1.getModelArtifactsInfoForJSON(modelArtifacts);
|
try {
|
this.LS.setItem(this.keys.info, JSON.stringify(modelArtifactsInfo));
|
this.LS.setItem(this.keys.topology, topology);
|
this.LS.setItem(this.keys.weightSpecs, weightSpecs);
|
this.LS.setItem(this.keys.weightData, io_utils_1.arrayBufferToBase64String(modelArtifacts.weightData));
|
this.LS.setItem(this.keys.modelMetadata, JSON.stringify({
|
format: modelArtifacts.format,
|
generatedBy: modelArtifacts.generatedBy,
|
convertedBy: modelArtifacts.convertedBy,
|
userDefinedMetadata: modelArtifacts.userDefinedMetadata
|
}));
|
return [2 /*return*/, { modelArtifactsInfo: modelArtifactsInfo }];
|
}
|
catch (err) {
|
// If saving failed, clean up all items saved so far.
|
this.LS.removeItem(this.keys.info);
|
this.LS.removeItem(this.keys.topology);
|
this.LS.removeItem(this.keys.weightSpecs);
|
this.LS.removeItem(this.keys.weightData);
|
this.LS.removeItem(this.keys.modelMetadata);
|
throw new Error("Failed to save model '" + this.modelPath + "' to local storage: " +
|
"size quota being exceeded is a possible cause of this failure: " +
|
("modelTopologyBytes=" + modelArtifactsInfo.modelTopologyBytes + ", ") +
|
("weightSpecsBytes=" + modelArtifactsInfo.weightSpecsBytes + ", ") +
|
("weightDataBytes=" + modelArtifactsInfo.weightDataBytes + "."));
|
}
|
}
|
return [2 /*return*/];
|
});
|
});
|
};
|
/**
|
* Load a model from local storage.
|
*
|
* See the documentation to `browserLocalStorage` for details on the saved
|
* artifacts.
|
*
|
* @returns The loaded model (if loading succeeds).
|
*/
|
BrowserLocalStorage.prototype.load = function () {
|
return __awaiter(this, void 0, void 0, function () {
|
var info, out, topology, weightSpecs, metadataString, metadata, weightDataBase64;
|
return __generator(this, function (_a) {
|
info = JSON.parse(this.LS.getItem(this.keys.info));
|
if (info == null) {
|
throw new Error("In local storage, there is no model with name '" + this.modelPath + "'");
|
}
|
if (info.modelTopologyType !== 'JSON') {
|
throw new Error('BrowserLocalStorage does not support loading non-JSON model ' +
|
'topology yet.');
|
}
|
out = {};
|
topology = JSON.parse(this.LS.getItem(this.keys.topology));
|
if (topology == null) {
|
throw new Error("In local storage, the topology of model '" + this.modelPath + "' " +
|
"is missing.");
|
}
|
out.modelTopology = topology;
|
weightSpecs = JSON.parse(this.LS.getItem(this.keys.weightSpecs));
|
if (weightSpecs == null) {
|
throw new Error("In local storage, the weight specs of model '" + this.modelPath + "' " +
|
"are missing.");
|
}
|
out.weightSpecs = weightSpecs;
|
metadataString = this.LS.getItem(this.keys.modelMetadata);
|
if (metadataString != null) {
|
metadata = JSON.parse(metadataString);
|
out.format = metadata['format'];
|
out.generatedBy = metadata['generatedBy'];
|
out.convertedBy = metadata['convertedBy'];
|
out.userDefinedMetadata = metadata['userDefinedMetadata'];
|
}
|
weightDataBase64 = this.LS.getItem(this.keys.weightData);
|
if (weightDataBase64 == null) {
|
throw new Error("In local storage, the binary weight values of model " +
|
("'" + this.modelPath + "' are missing."));
|
}
|
out.weightData = io_utils_1.base64StringToArrayBuffer(weightDataBase64);
|
return [2 /*return*/, out];
|
});
|
});
|
};
|
BrowserLocalStorage.URL_SCHEME = 'localstorage://';
|
return BrowserLocalStorage;
|
}());
|
exports.BrowserLocalStorage = BrowserLocalStorage;
|
exports.localStorageRouter = function (url) {
|
if (!environment_1.env().getBool('IS_BROWSER')) {
|
return null;
|
}
|
else {
|
if (!Array.isArray(url) && url.startsWith(BrowserLocalStorage.URL_SCHEME)) {
|
return browserLocalStorage(url.slice(BrowserLocalStorage.URL_SCHEME.length));
|
}
|
else {
|
return null;
|
}
|
}
|
};
|
router_registry_1.IORouterRegistry.registerSaveRouter(exports.localStorageRouter);
|
router_registry_1.IORouterRegistry.registerLoadRouter(exports.localStorageRouter);
|
/**
|
* Factory function for local storage IOHandler.
|
*
|
* This `IOHandler` supports both `save` and `load`.
|
*
|
* For each model's saved artifacts, four items are saved to local storage.
|
* - `${PATH_SEPARATOR}/${modelPath}/info`: Contains meta-info about the
|
* model, such as date saved, type of the topology, size in bytes, etc.
|
* - `${PATH_SEPARATOR}/${modelPath}/topology`: Model topology. For Keras-
|
* style models, this is a stringized JSON.
|
* - `${PATH_SEPARATOR}/${modelPath}/weight_specs`: Weight specs of the
|
* model, can be used to decode the saved binary weight values (see
|
* item below).
|
* - `${PATH_SEPARATOR}/${modelPath}/weight_data`: Concatenated binary
|
* weight values, stored as a base64-encoded string.
|
*
|
* Saving may throw an `Error` if the total size of the artifacts exceed the
|
* browser-specific quota.
|
*
|
* @param modelPath A unique identifier for the model to be saved. Must be a
|
* non-empty string.
|
* @returns An instance of `IOHandler`, which can be used with, e.g.,
|
* `tf.Model.save`.
|
*/
|
function browserLocalStorage(modelPath) {
|
return new BrowserLocalStorage(modelPath);
|
}
|
exports.browserLocalStorage = browserLocalStorage;
|
var BrowserLocalStorageManager = /** @class */ (function () {
|
function BrowserLocalStorageManager() {
|
util_1.assert(environment_1.env().getBool('IS_BROWSER'), function () { return 'Current environment is not a web browser'; });
|
util_1.assert(typeof window === 'undefined' ||
|
typeof window.localStorage !== 'undefined', function () { return 'Current browser does not appear to support localStorage'; });
|
this.LS = window.localStorage;
|
}
|
BrowserLocalStorageManager.prototype.listModels = function () {
|
return __awaiter(this, void 0, void 0, function () {
|
var out, prefix, suffix, i, key, modelPath;
|
return __generator(this, function (_a) {
|
out = {};
|
prefix = PATH_PREFIX + PATH_SEPARATOR;
|
suffix = PATH_SEPARATOR + INFO_SUFFIX;
|
for (i = 0; i < this.LS.length; ++i) {
|
key = this.LS.key(i);
|
if (key.startsWith(prefix) && key.endsWith(suffix)) {
|
modelPath = getModelPathFromKey(key);
|
out[modelPath] = JSON.parse(this.LS.getItem(key));
|
}
|
}
|
return [2 /*return*/, out];
|
});
|
});
|
};
|
BrowserLocalStorageManager.prototype.removeModel = function (path) {
|
return __awaiter(this, void 0, void 0, function () {
|
var keys, info;
|
return __generator(this, function (_a) {
|
path = maybeStripScheme(path);
|
keys = getModelKeys(path);
|
if (this.LS.getItem(keys.info) == null) {
|
throw new Error("Cannot find model at path '" + path + "'");
|
}
|
info = JSON.parse(this.LS.getItem(keys.info));
|
this.LS.removeItem(keys.info);
|
this.LS.removeItem(keys.topology);
|
this.LS.removeItem(keys.weightSpecs);
|
this.LS.removeItem(keys.weightData);
|
return [2 /*return*/, info];
|
});
|
});
|
};
|
return BrowserLocalStorageManager;
|
}());
|
exports.BrowserLocalStorageManager = BrowserLocalStorageManager;
|
if (environment_1.env().getBool('IS_BROWSER')) {
|
// Wrap the construction and registration, to guard against browsers that
|
// don't support Local Storage.
|
try {
|
model_management_1.ModelStoreManagerRegistry.registerManager(BrowserLocalStorage.URL_SCHEME, new BrowserLocalStorageManager());
|
}
|
catch (err) {
|
}
|
}
|
//# sourceMappingURL=local_storage.js.map
|
