blace::ml_core::ModelInferenceArgs Struct Reference

#include <types.h>

Public Member Functions
ml_core::BlaceHash	hash ()
bool	operator== (const ModelInferenceArgs &other) const
bool	operator!= (const ModelInferenceArgs &other) const

Public Attributes
std::vector< Backend >	backends
int	deprecated = true
int	seed = 0
int	run_threaded = false
int	plot_inputs = false
int	gpu_mem_opti = false
int	empty_cache_after_inference = false
int	experimental_torchscript_autocast = false

Detailed Description

Model inference arguments.

Member Function Documentation

hash()

ml_core::BlaceHash blace::ml_core::ModelInferenceArgs::hash

(

)

Hash the struct.

Returns

operator!=()

bool blace::ml_core::ModelInferenceArgs::operator!= ( const ModelInferenceArgs &  other ) const

inline

Simple inequality operator.

Parameters

other

The other struct.

Returns

operator==()

bool blace::ml_core::ModelInferenceArgs::operator==

(

const ModelInferenceArgs &

other

)

const

Simple equality operator.

Parameters

other

The other struct.

Returns

Member Data Documentation

backends

std::vector<Backend> blace::ml_core::ModelInferenceArgs::backends

The backends to run the inference on. It will run the execution with the first backend from the list supported by the model.

deprecated

int blace::ml_core::ModelInferenceArgs::deprecated = true

Inference in half (fp16) precision if possible.

empty_cache_after_inference

int blace::ml_core::ModelInferenceArgs::empty_cache_after_inference = false

Empty backend (cuda or metal) caches after every inference, might save some memory.

experimental_torchscript_autocast

int blace::ml_core::ModelInferenceArgs::experimental_torchscript_autocast = false

Run torchscript model in autocast mode.

gpu_mem_opti

int blace::ml_core::ModelInferenceArgs::gpu_mem_opti = false

If several models are invoked in computation, unload all models to the cpu and have only current model on hardware accelerator.

plot_inputs

int blace::ml_core::ModelInferenceArgs::plot_inputs = false

Not used.

run_threaded

int blace::ml_core::ModelInferenceArgs::run_threaded = false

Run inference in a seperate thread which can be cancelled (not available in beta).

seed

int blace::ml_core::ModelInferenceArgs::seed = 0

A value to seed random operators with.