@InProceedings{Rebensburg2023_1170,
author = {Mika Rebensburg and Stefan Hillmann and Nils Feldhus},
booktitle = {Studientexte zur Sprachkommunikation: Elektronische Sprachsignalverarbeitung 2023},
title = {Automatic User Experience Evaluation of Goal-Oriented Dialogs Using Pre- Trained Language Models},
year = {2023},
editor = {Christoph Draxler},
month = mar,
pages = {32--39},
publisher = {TUDpress, Dresden},
abstract = {Dialog evaluation methods based on Pre-trained Language Models
(Pr-LMs) have been primarily used for open-domain dialogs with the goal of comparing
systems in terms of dialog skills relevant in casual chats, such as naturalness,
engagement, and relevance. Automatic evaluation metrics for goal-oriented and
closed-domain dialogs often measure few and objective metrics like task success
rate and ignore subjective aspects of the User Experience (UX). Important subjective
usability aspects like satisfaction go beyond simple objective metrics and have
traditionally been assessed using questionnaires in an experimental setup. Information
about subjective UX is often implicitly contained in the dialog text which
could therefore be used to estimate the true UX in an automated fashion using
Pr-LMs. This works aims to explore automatic text-based and multifaceted UX
evaluation of goal-oriented chatbot interactions using Pr-LMs. We examine both a
supervised learning approach and an approach based on an automatic, referencefree
and unsupervised dialog evaluation metric. With supervised learning, we train
a Pr-LM that predicts several relevant UX aspects with moderate correlation values.
SimCSE embeddings perform best and even outperform the UX ratings of human
observers collected in a previous study. While the reference-free approach manages
to achieve low to moderate correlations, we suspect that this method mainly
exploits the correlation between dialog length and user satisfaction and could hence
fail in scenarios where these are not correlated.},
isbn = {978-3-95908-303-4},
issn = {0940-6832},
keywords = {Interaction & Dialogue},
url = {https://www.essv.de/pdf/2023_32_39.pdf},
}