gx

chenyc

2025-02-12 ea42ff3ebee1eeb3fb29423aa848a249441db81c

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
/**
 * @license
 * Copyright 2023 Google LLC.
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 * =============================================================================
 */
/* Original source: keras-nlp/models/preprocessor.py */
import { serialization } from '@tensorflow/tfjs-core';
import { Layer } from '../../../engine/topology';
import { Tokenizer } from '../tokenizers';
import { deserializeKerasObject, serializeKerasObject } from '../../../utils/generic_utils';
/**
 * Base class for model Preprocessors.
 */
class Preprocessor extends Layer {
    constructor(args) {
        super(args);
    }
    /**
     * The tokenizer used to tokenize strings.
     */
    get tokenizer() {
        return this._tokenizer;
    }
    set tokenizer(value) {
        this._tokenizer = value;
    }
    getConfig() {
        const config = super.getConfig();
        config.tokenizer = serializeKerasObject(this.tokenizer);
        return config;
    }
    static fromConfig(cls, config) {
        const kwargs = config;
        if (config.tokenizer != null && !(config.tokenizer instanceof Tokenizer)) {
            const tokenizerConfigDict = config.tokenizer;
            kwargs.tokenizer = deserializeKerasObject(tokenizerConfigDict, serialization.SerializationMap.getMap().classNameMap, {}, 'preprocessor');
        }
        return new cls(kwargs);
    }
    static tokenizerCls(cls) { }
}
/** @nocollapse */
Preprocessor.className = 'Preprocessor';
export { Preprocessor };
serialization.registerClass(Preprocessor);
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoicHJlcHJvY2Vzc29yLmpzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vLi4vLi4vLi4vLi4vLi4vLi4vLi4vdGZqcy1sYXllcnMvc3JjL2xheWVycy9ubHAvbW9kZWxzL3ByZXByb2Nlc3Nvci50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFBQTs7Ozs7Ozs7Ozs7Ozs7O0dBZUc7QUFFSCx1REFBdUQ7QUFDdkQsT0FBTyxFQUFFLGFBQWEsRUFBRSxNQUFNLHVCQUF1QixDQUFDO0FBRXRELE9BQU8sRUFBRSxLQUFLLEVBQWEsTUFBTSwwQkFBMEIsQ0FBQztBQUM1RCxPQUFPLEVBQUUsU0FBUyxFQUFFLE1BQU0sZUFBZSxDQUFDO0FBRTFDLE9BQU8sRUFBRSxzQkFBc0IsRUFBRSxvQkFBb0IsRUFBRSxNQUFNLDhCQUE4QixDQUFDO0FBRTVGOztHQUVHO0FBQ0gsTUFBYSxZQUFhLFNBQVEsS0FBSztJQU1yQyxZQUFZLElBQWU7UUFDekIsS0FBSyxDQUFDLElBQUksQ0FBQyxDQUFDO0lBQ2QsQ0FBQztJQUVEOztPQUVHO0lBQ0gsSUFBSSxTQUFTO1FBQ1gsT0FBTyxJQUFJLENBQUMsVUFBVSxDQUFDO0lBQ3pCLENBQUM7SUFFRCxJQUFJLFNBQVMsQ0FBQyxLQUFnQjtRQUM1QixJQUFJLENBQUMsVUFBVSxHQUFHLEtBQUssQ0FBQztJQUMxQixDQUFDO0lBRVEsU0FBUztRQUNoQixNQUFNLE1BQU0sR0FBRyxLQUFLLENBQUMsU0FBUyxFQUFFLENBQUM7UUFDakMsTUFBTSxDQUFDLFNBQVMsR0FBRyxvQkFBb0IsQ0FBQyxJQUFJLENBQUMsU0FBUyxDQUFDLENBQUM7UUFDeEQsT0FBTyxNQUFNLENBQUM7SUFDaEIsQ0FBQztJQUVELE1BQU0sQ0FBVSxVQUFVLENBQ3hCLEdBQTZDLEVBQzdDLE1BQWdDO1FBRWhDLE1BQU0sTUFBTSxHQUFXLE1BQU0sQ0FBQztRQUU5QixJQUFJLE1BQU0sQ0FBQyxTQUFTLElBQUksSUFBSSxJQUFJLENBQUMsQ0FBQyxNQUFNLENBQUMsU0FBUyxZQUFZLFNBQVMsQ0FBQyxFQUFFO1lBQ3hFLE1BQU0sbUJBQW1CLEdBQUcsTUFBTSxDQUFDLFNBQXFDLENBQUM7WUFFekUsTUFBTSxDQUFDLFNBQVMsR0FBRyxzQkFBc0IsQ0FDdkMsbUJBQW1CLEVBQ25CLGFBQWEsQ0FBQyxnQkFBZ0IsQ0FBQyxNQUFNLEVBQUUsQ0FBQyxZQUFZLEVBQ3BELEVBQUUsRUFBRSxjQUFjLENBQUMsQ0FBQztTQUN2QjtRQUNELE9BQU8sSUFBSSxHQUFHLENBQUMsTUFBTSxDQUFDLENBQUM7SUFDekIsQ0FBQztJQUVELE1BQU0sQ0FBQyxZQUFZLENBQ2pCLEdBQTZDLElBQUcsQ0FBQzs7QUE1Q25ELGtCQUFrQjtBQUNYLHNCQUFTLEdBQUcsY0FBYyxDQUFDO1NBRnZCLFlBQVk7QUErQ3pCLGFBQWEsQ0FBQyxhQUFhLENBQUMsWUFBWSxDQUFDLENBQUMiLCJzb3VyY2VzQ29udGVudCI6WyIvKipcbiAqIEBsaWNlbnNlXG4gKiBDb3B5cmlnaHQgMjAyMyBHb29nbGUgTExDLlxuICogTGljZW5zZWQgdW5kZXIgdGhlIEFwYWNoZSBMaWNlbnNlLCBWZXJzaW9uIDIuMCAodGhlIFwiTGljZW5zZVwiKTtcbiAqIHlvdSBtYXkgbm90IHVzZSB0aGlzIGZpbGUgZXhjZXB0IGluIGNvbXBsaWFuY2Ugd2l0aCB0aGUgTGljZW5zZS5cbiAqIFlvdSBtYXkgb2J0YWluIGEgY29weSBvZiB0aGUgTGljZW5zZSBhdFxuICpcbiAqIGh0dHA6Ly93d3cuYXBhY2hlLm9yZy9saWNlbnNlcy9MSUNFTlNFLTIuMFxuICpcbiAqIFVubGVzcyByZXF1aXJlZCBieSBhcHBsaWNhYmxlIGxhdyBvciBhZ3JlZWQgdG8gaW4gd3JpdGluZywgc29mdHdhcmVcbiAqIGRpc3RyaWJ1dGVkIHVuZGVyIHRoZSBMaWNlbnNlIGlzIGRpc3RyaWJ1dGVkIG9uIGFuIFwiQVMgSVNcIiBCQVNJUyxcbiAqIFdJVEhPVVQgV0FSUkFOVElFUyBPUiBDT05ESVRJT05TIE9GIEFOWSBLSU5ELCBlaXRoZXIgZXhwcmVzcyBvciBpbXBsaWVkLlxuICogU2VlIHRoZSBMaWNlbnNlIGZvciB0aGUgc3BlY2lmaWMgbGFuZ3VhZ2UgZ292ZXJuaW5nIHBlcm1pc3Npb25zIGFuZFxuICogbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuXG4gKiA9PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PVxuICovXG5cbi8qIE9yaWdpbmFsIHNvdXJjZToga2VyYXMtbmxwL21vZGVscy9wcmVwcm9jZXNzb3IucHkgKi9cbmltcG9ydCB7IHNlcmlhbGl6YXRpb24gfSBmcm9tICdAdGVuc29yZmxvdy90ZmpzLWNvcmUnO1xuXG5pbXBvcnQgeyBMYXllciwgTGF5ZXJBcmdzIH0gZnJvbSAnLi4vLi4vLi4vZW5naW5lL3RvcG9sb2d5JztcbmltcG9ydCB7IFRva2VuaXplciB9IGZyb20gJy4uL3Rva2VuaXplcnMnO1xuaW1wb3J0IHsgS3dhcmdzIH0gZnJvbSAnLi4vLi4vLi4vdHlwZXMnO1xuaW1wb3J0IHsgZGVzZXJpYWxpemVLZXJhc09iamVjdCwgc2VyaWFsaXplS2VyYXNPYmplY3QgfSBmcm9tICcuLi8uLi8uLi91dGlscy9nZW5lcmljX3V0aWxzJztcblxuLyoqXG4gKiBCYXNlIGNsYXNzIGZvciBtb2RlbCBQcmVwcm9jZXNzb3JzLlxuICovXG5leHBvcnQgY2xhc3MgUHJlcHJvY2Vzc29yIGV4dGVuZHMgTGF5ZXIge1xuICAvKiogQG5vY29sbGFwc2UgKi9cbiAgc3RhdGljIGNsYXNzTmFtZSA9ICdQcmVwcm9jZXNzb3InO1xuXG4gIHByaXZhdGUgX3Rva2VuaXplcjogVG9rZW5pemVyO1xuXG4gIGNvbnN0cnVjdG9yKGFyZ3M6IExheWVyQXJncykge1xuICAgIHN1cGVyKGFyZ3MpO1xuICB9XG5cbiAgLyoqXG4gICAqIFRoZSB0b2tlbml6ZXIgdXNlZCB0byB0b2tlbml6ZSBzdHJpbmdzLlxuICAgKi9cbiAgZ2V0IHRva2VuaXplcigpIHtcbiAgICByZXR1cm4gdGhpcy5fdG9rZW5pemVyO1xuICB9XG5cbiAgc2V0IHRva2VuaXplcih2YWx1ZTogVG9rZW5pemVyKSB7XG4gICAgdGhpcy5fdG9rZW5pemVyID0gdmFsdWU7XG4gIH1cblxuICBvdmVycmlkZSBnZXRDb25maWcoKTogc2VyaWFsaXphdGlvbi5Db25maWdEaWN0IHtcbiAgICBjb25zdCBjb25maWcgPSBzdXBlci5nZXRDb25maWcoKTtcbiAgICBjb25maWcudG9rZW5pemVyID0gc2VyaWFsaXplS2VyYXNPYmplY3QodGhpcy50b2tlbml6ZXIpO1xuICAgIHJldHVybiBjb25maWc7XG4gIH1cblxuICBzdGF0aWMgb3ZlcnJpZGUgZnJvbUNvbmZpZzxUIGV4dGVuZHMgc2VyaWFsaXphdGlvbi5TZXJpYWxpemFibGU+KFxuICAgIGNsczogc2VyaWFsaXphdGlvbi5TZXJpYWxpemFibGVDb25zdHJ1Y3RvcjxUPixcbiAgICBjb25maWc6IHNlcmlhbGl6YXRpb24uQ29uZmlnRGljdFxuICApOiBUIHtcbiAgICBjb25zdCBrd2FyZ3M6IEt3YXJncyA9IGNvbmZpZztcblxuICAgIGlmIChjb25maWcudG9rZW5pemVyICE9IG51bGwgJiYgIShjb25maWcudG9rZW5pemVyIGluc3RhbmNlb2YgVG9rZW5pemVyKSkge1xuICAgICAgY29uc3QgdG9rZW5pemVyQ29uZmlnRGljdCA9IGNvbmZpZy50b2tlbml6ZXIgYXMgc2VyaWFsaXphdGlvbi5Db25maWdEaWN0O1xuXG4gICAgICBrd2FyZ3MudG9rZW5pemVyID0gZGVzZXJpYWxpemVLZXJhc09iamVjdChcbiAgICAgICAgdG9rZW5pemVyQ29uZmlnRGljdCxcbiAgICAgICAgc2VyaWFsaXphdGlvbi5TZXJpYWxpemF0aW9uTWFwLmdldE1hcCgpLmNsYXNzTmFtZU1hcCxcbiAgICAgICAge30sICdwcmVwcm9jZXNzb3InKTtcbiAgICB9XG4gICAgcmV0dXJuIG5ldyBjbHMoa3dhcmdzKTtcbiAgfVxuXG4gIHN0YXRpYyB0b2tlbml6ZXJDbHM8VCBleHRlbmRzIHNlcmlhbGl6YXRpb24uU2VyaWFsaXphYmxlPihcbiAgICBjbHM6IHNlcmlhbGl6YXRpb24uU2VyaWFsaXphYmxlQ29uc3RydWN0b3I8VD4pIHt9XG59XG5zZXJpYWxpemF0aW9uLnJlZ2lzdGVyQ2xhc3MoUHJlcHJvY2Vzc29yKTtcbiJdfQ==