2022
Huertas-Tato, Javier; Martín, Alejandro; Camacho, David
SILT: Efficient transformer training for inter-lingual inference Journal Article
In: Expert Systems with Applications, vol. 200, pp. 116923, 2022, ISSN: 0957-4174.
Abstract | Links | BibTeX | Tags: Deep Learning, Disinformation, Embeddings, Natural language inference, Natural Language Processing, Sentence alignment, Transformers
@article{huertas-tato_silt_2022,
title = {SILT: Efficient transformer training for inter-lingual inference},
author = {Javier Huertas-Tato and Alejandro Martín and David Camacho},
url = {https://www.sciencedirect.com/science/article/pii/S0957417422003578},
doi = {10.1016/j.eswa.2022.116923},
issn = {0957-4174},
year = {2022},
date = {2022-08-01},
urldate = {2022-08-01},
journal = {Expert Systems with Applications},
volume = {200},
pages = {116923},
abstract = {The ability of transformers to perform precision tasks such as question answering, Natural Language Inference (NLI) or summarizing, has enabled them to be ranked as one of the best paradigms to address Natural Language Processing (NLP) tasks. NLI is one of the best scenarios to test these architectures, due to the knowledge required to understand complex sentences and established relationships between a hypothesis and a premise. Nevertheless, these models suffer from the incapacity to generalize to other domains or from difficulties to face multilingual and interlingual scenarios. The leading pathway in the literature to address these issues involve designing and training extremely large architectures, but this causes unpredictable behaviors and establishes barriers which impede broad access and fine tuning. In this paper, we propose a new architecture called Siamese Inter-Lingual Transformer (SILT). This architecture is able to efficiently align multilingual embeddings for Natural Language Inference, allowing for unmatched language pairs to be processed. SILT leverages siamese pre-trained multi-lingual transformers with frozen weights where the two input sentences attend to each other to later be combined through a matrix alignment method. The experimental results carried out in this paper evidence that SILT allows to reduce drastically the number of trainable parameters while allowing for inter-lingual NLI and achieving state-of-the-art performance on common benchmarks.},
keywords = {Deep Learning, Disinformation, Embeddings, Natural language inference, Natural Language Processing, Sentence alignment, Transformers},
pubstate = {published},
tppubtype = {article}
}
The ability of transformers to perform precision tasks such as question answering, Natural Language Inference (NLI) or summarizing, has enabled them to be ranked as one of the best paradigms to address Natural Language Processing (NLP) tasks. NLI is one of the best scenarios to test these architectures, due to the knowledge required to understand complex sentences and established relationships between a hypothesis and a premise. Nevertheless, these models suffer from the incapacity to generalize to other domains or from difficulties to face multilingual and interlingual scenarios. The leading pathway in the literature to address these issues involve designing and training extremely large architectures, but this causes unpredictable behaviors and establishes barriers which impede broad access and fine tuning. In this paper, we propose a new architecture called Siamese Inter-Lingual Transformer (SILT). This architecture is able to efficiently align multilingual embeddings for Natural Language Inference, allowing for unmatched language pairs to be processed. SILT leverages siamese pre-trained multi-lingual transformers with frozen weights where the two input sentences attend to each other to later be combined through a matrix alignment method. The experimental results carried out in this paper evidence that SILT allows to reduce drastically the number of trainable parameters while allowing for inter-lingual NLI and achieving state-of-the-art performance on common benchmarks.