How to use ViLLM?

Running Pipeline

Run on local computer

vieval --mode generation \
           --model_name ura-hcmut/MixSUra \
           --dataset_name zalo_e2eqa \
           --prompting_strategy 0 \
           --fewshot_prompting True \
           --seed 42

Run on TGI

vieval --mode generation \
           --model_name ura-hcmut/MixSUra \
           --dataset_name zalo_e2eqa \
           --prompting_strategy 0 \
           --fewshot_prompting True \
           --seed 42 \
           --tgi http://127.0.0.1:10025

Run on GPT (gpt-3.5-turbo, gpt-4)

vieval --mode generation \
           --model_name gpt-4 \
           --dataset_name zalo_e2eqa \
           --prompting_strategy 0 \
           --fewshot_prompting True \
           --seed 42

Evaluation

vieval --mode evaluation \
           --model_name gpt-4 \
           --dataset_name zalo_e2eqa \
           --prompting_strategy 0 \
           --fewshot_prompting True \
           --seed 42 \
           --output_dir results \
           --out_eval_dir out_new

End2End Pipeline

vieval --mode end2end \
           --model_name gpt-4 \
           --dataset_name zalo_e2eqa \
           --prompting_strategy 0 \
           --fewshot_prompting True \
           --seed 42 \
           --output_dir results \
           --out_eval_dir out_new