diff --git a/source/executorch.js b/source/executorch.js index b6f13f9188..5b72ab0d95 100644 --- a/source/executorch.js +++ b/source/executorch.js @@ -3,16 +3,17 @@ const executorch = {}; +import * as flatbuffers from './flatbuffers.js'; import * as python from './python.js'; import * as pytorch from './pytorch.js'; executorch.ModelFactory = class { match(context) { - const reader = context.peek('flatbuffers.binary'); - if (reader && reader.identifier === 'ET12') { + const container = executorch.Container.open(context); + if (container) { context.type = 'executorch'; - context.target = reader; + context.target = container; } } @@ -22,20 +23,20 @@ executorch.ModelFactory = class { const metadata = await pytorch.Metadata.open(context); const execution = new python.Execution(); metadata.register(execution); - const reader = context.target; - const program = executorch.schema.Program.create(reader); - return new executorch.Model(execution, program); + const target = context.target; + await target.read(); + return new executorch.Model(execution, target); } }; executorch.Model = class { - constructor(execution, program) { - this.format = `ExecuTorch v${program.version}`; + constructor(execution, context) { + this.format = `ExecuTorch v${context.program.version}`; this.graphs = []; - for (const plan of program.execution_plan) { + for (const plan of context.program.execution_plan) { for (const chain of plan.chains) { - const graph = new executorch.Graph(execution, chain, plan); + const graph = new executorch.Graph(execution, context, plan, chain); this.graphs.push(graph); } } @@ -44,7 +45,7 @@ executorch.Model = class { executorch.Graph = class { - constructor(execution, chain, plan) { + constructor(execution, context, plan, chain) { this.inputs = []; this.outputs = []; this.nodes = []; @@ -52,7 +53,7 @@ executorch.Graph = class { values.map = (index, output) => { if (!values.has(index)) { const v = plan.values[index].val; - const tensor = v instanceof executorch.schema.Tensor || v instanceof executorch.schema.TensorList; + const tensor = v instanceof executorch.schema.Tensor || v instanceof executorch.schema.TensorList || v instanceof executorch.schema.OptionalTensorList; if (output && !tensor) { const value = [new executorch.Value(index.toString(), null, null)]; values.set(index, { type: null, value }); @@ -64,7 +65,7 @@ executorch.Graph = class { const type = new executorch.TensorType(tensor); let initializer = null; if (v.data_buffer_idx > 0) { - initializer = new executorch.Tensor(tensor); + initializer = new executorch.Tensor(tensor, context); } const identifier = tensors.length > 1 ? `${index}.${i}` : index.toString(); const value = new executorch.Value(identifier, type, initializer); @@ -99,7 +100,7 @@ executorch.Graph = class { this.outputs.push(argument); } for (const instruction of chain.instructions) { - const node = new executorch.Node(execution, instruction, plan, values); + const node = new executorch.Node(execution, context, plan, chain, instruction, values); this.nodes.push(node); } } @@ -129,7 +130,7 @@ executorch.Value = class Value { executorch.Node = class { - constructor(execution, instruction, plan, values) { + constructor(execution, context, plan, chain, instruction, values) { this.name = ''; this.inputs = []; this.outputs = []; @@ -179,12 +180,31 @@ executorch.Node = class { const args = instr_args.args; const name = delegate.id; this.type = { name }; + let data = null; + const DataLocation = executorch.schema.DataLocation; + switch (delegate.processed.location) { + case DataLocation.INLINE: { + data = context.program.backend_delegate_data[delegate.processed.index].data; + break; + } + case DataLocation.SEGMENT: { + const segment = context.program.segments[delegate.processed.index]; + context.reader.seek(context.extended_file_header.segment_base_offset + segment.offset.toNumber()); + data = context.reader.read(segment.size.toNumber()); + break; + } + default: { + throw new executorch.Error(`Delegate data location '${delegate.processed.location}' not implemented.`); + } + } switch (name) { case 'XnnpackBackend': { const input = values.map(args[0]); const output = values.map(args[1], true); this.inputs.push(new executorch.Argument('input', input.value, input.type)); this.outputs.push(new executorch.Argument('output', output.value, output.type)); + flatbuffers.BinaryReader.open(data); + // executorch/backends/xnnpack/serialization/schema.fbs break; } case 'CoreMLBackend': { @@ -227,7 +247,7 @@ executorch.TensorType = class { if (tensor.scalar_type >= executorch.TensorType._types.length) { throw new executorch.Error(`Unknown tensor data type '${tensor.scalar_type}'.`); } - this.dataType = executorch.TensorType._types.length[tensor.scalar_type]; + this.dataType = executorch.TensorType._types[tensor.scalar_type]; this.shape = new executorch.TensorShape(Array.from(tensor.sizes)); } @@ -252,8 +272,60 @@ executorch.TensorShape = class { executorch.Tensor = class { - constructor(tensor) { + constructor(tensor, context) { this.type = new executorch.TensorType(tensor); + const data_buffer_idx = tensor.data_buffer_idx; + if (tensor.extra_tensor_info) { + throw new executorch.Error('Extra tensor info not implemented.'); + } else if (context.program.constant_buffers) { + throw new executorch.Error('Constant buffers not implemented.'); + } else if (tensor.allocation_info === null) { + const constant_segment = context.program.constant_segment; + const data_segment = context.program.segments[constant_segment.segment_index]; + const offset = constant_segment.offsets[data_buffer_idx].toNumber(); + const next = data_buffer_idx + 1 < constant_segment.offsets.length ? constant_segment.offsets[data_buffer_idx + 1].toNumber() : data_segment.size.toNumber(); + const size = next - offset; + const reader = context.reader; + reader.seek(context.extended_file_header.segment_base_offset + data_segment.offset.toNumber() + offset); + this.encoding = '<'; + this.values = reader.read(size); + reader.seek(0); + } else { + throw new executorch.Error('Tensor allocation info not implemented.'); + } + } +}; + +executorch.Container = class { + + static open(context) { + const reader = context.peek('flatbuffers.binary'); + if (reader && reader.identifier === 'ET12') { + return new executorch.Container(context, reader); + } + return null; + } + + constructor(context, reader) { + this.context = context; + this.reader = reader; + } + + async read() { + this.program = executorch.schema.Program.create(this.reader); + this.reader = this.context.read('binary'); + if (this.reader.length >= 32) { + this.reader.seek(8); + const magic = String.fromCharCode(...this.reader.read(4)); + if (magic === 'eh00') { + this.extended_file_header = { + length: this.reader.uint32(), + program_size: this.reader.uint64().toNumber(), + segment_base_offset: this.reader.uint64().toNumber(), + }; + } + this.reader.seek(0); + } } }; diff --git a/test/models.json b/test/models.json index 9d4aefe6b7..a6b62d6039 100644 --- a/test/models.json +++ b/test/models.json @@ -6570,6 +6570,20 @@ "format": "PyTorch v1.6", "link": "https://github.com/lutzroeder/netron/issues/543" }, + { + "type": "pytorch", + "target": "style_transfer_candy_coreml.pte", + "source": "https://github.com/user-attachments/files/18469498/style_transfer_candy_coreml.pte.zip[style_transfer_candy_coreml.pte]", + "format": "ExecuTorch v0", + "link": "https://github.com/lutzroeder/netron/issues/1175" + }, + { + "type": "pytorch", + "target": "style_transfer_candy_xnnpack.pte", + "source": "https://github.com/user-attachments/files/18469499/style_transfer_candy_xnnpack.pte.zip[style_transfer_candy_xnnpack.pte]", + "format": "ExecuTorch v0", + "link": "https://github.com/lutzroeder/netron/issues/1175" + }, { "type": "pytorch", "target": "superpoint.pt", diff --git a/tools/executorch b/tools/executorch index 6c42b58d3f..fc3ac4c91f 100755 --- a/tools/executorch +++ b/tools/executorch @@ -15,9 +15,11 @@ clean() { sync() { echo "executorch sync" - mkdir -p "./third_party/source/executorch/schema" - curl --silent --location --output "./third_party/source/executorch/schema/scalar_type.fbs" "https://github.com/pytorch/executorch/raw/main/schema/scalar_type.fbs" - curl --silent --location --output "./third_party/source/executorch/schema/program.fbs" "https://github.com/pytorch/executorch/raw/main/schema/program.fbs" + [ -d "./third_party/source/executorch" ] || git clone --quiet https://github.com/pytorch/executorch.git "./third_party/source/executorch" + git -C "./third_party/source/executorch" pull --quiet --prune + # mkdir -p "./third_party/source/executorch/schema" + # curl --silent --location --output "./third_party/source/executorch/schema/scalar_type.fbs" "https://github.com/pytorch/executorch/raw/main/schema/scalar_type.fbs" + # curl --silent --location --output "./third_party/source/executorch/schema/program.fbs" "https://github.com/pytorch/executorch/raw/main/schema/program.fbs" } schema() {