2023
Automatic blight disease detection in potato (Solanum tuberosum L.) and tomato (Solanum lycopersicum, L. 1753) plants using deep learning Journal Article
Anim-Ayeko, Alberta Odamea; Schillaci, Calogero; Lipani, Aldo
In: Smart Agricultural Technology, vol. 4, pp. 100178, 2023, ISSN: 2772-3755.
@article{ANIMAYEKO2023100178,
title = {Automatic blight disease detection in potato (Solanum tuberosum L.) and tomato (Solanum lycopersicum, L. 1753) plants using deep learning},
author = {Alberta Odamea Anim-Ayeko and Calogero Schillaci and Aldo Lipani},
url = {https://www.sciencedirect.com/science/article/pii/S2772375523000084},
doi = {https://doi.org/10.1016/j.atech.2023.100178},
issn = {2772-3755},
year = {2023},
date = {2023-01-01},
journal = {Smart Agricultural Technology},
volume = {4},
pages = {100178},
abstract = {Early and late blight are two diseases which pose a huge risk to both potato (Solanum tuberosum L.) and tomato (Solanum lycopersicum, L. 1753) crops and make farmers run at a loss. The early and automatic detection of these diseases would save time as well as enable farmers to act quickly on crops which have been affected. Machine learning and deep learning technology provide many solutions for the detection of the blight diseases in affected crops, and are common in the literature. However, explanation methods for such solutions are not common, but are necessary, considering some machine learning models are seen as black boxes. This study proposes a ResNet-9 model which detects the blight disease state of potato and tomato leaf images, which farmers can leverage. With the data obtained from the popular “Plant Village Dataset”, there were 3,990 initial training data samples. After augmenting the training set and a rigorous hyperparameter optimization procedure, the model was trained with these hyperparameter values, and examined on the test set, which contained 1,331 images. A test accuracy of 99.25%, 99.67% overall precision, 99.33% overall recall and 99.33% overall F1-score values were achieved. To fully understand the model, explanations for the proposed model were provided through saliency maps, which showed the reasoning behind the predictions of the model. It was observed that the ResNet-9 model considered the shape of the leaf, diseased areas present and general green areas of the leaf for its predictions and this makes us understand the model predictions better and see that the model behaves as expected. Our results could contribute to the testing and deployment of Convolutional Neural Network (CNN) models for classification of proximal sensing images of potato (Solanum tuberosum L.) and tomato (Solanum lycopersicum, L. 1753) plant leaves. Further studies would benefit from this modeling framework and would have the chance to test several other variables to determine the leaf infections in an earlier stage for crop protection.},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
Early and late blight are two diseases which pose a huge risk to both potato (Solanum tuberosum L.) and tomato (Solanum lycopersicum, L. 1753) crops and make farmers run at a loss. The early and automatic detection of these diseases would save time as well as enable farmers to act quickly on crops which have been affected. Machine learning and deep learning technology provide many solutions for the detection of the blight diseases in affected crops, and are common in the literature. However, explanation methods for such solutions are not common, but are necessary, considering some machine learning models are seen as black boxes. This study proposes a ResNet-9 model which detects the blight disease state of potato and tomato leaf images, which farmers can leverage. With the data obtained from the popular “Plant Village Dataset”, there were 3,990 initial training data samples. After augmenting the training set and a rigorous hyperparameter optimization procedure, the model was trained with these hyperparameter values, and examined on the test set, which contained 1,331 images. A test accuracy of 99.25%, 99.67% overall precision, 99.33% overall recall and 99.33% overall F1-score values were achieved. To fully understand the model, explanations for the proposed model were provided through saliency maps, which showed the reasoning behind the predictions of the model. It was observed that the ResNet-9 model considered the shape of the leaf, diseased areas present and general green areas of the leaf for its predictions and this makes us understand the model predictions better and see that the model behaves as expected. Our results could contribute to the testing and deployment of Convolutional Neural Network (CNN) models for classification of proximal sensing images of potato (Solanum tuberosum L.) and tomato (Solanum lycopersicum, L. 1753) plant leaves. Further studies would benefit from this modeling framework and would have the chance to test several other variables to determine the leaf infections in an earlier stage for crop protection.
2022
Tab this Folder of Documents: Page Stream Segmentation of Business Documents Inproceedings 🏆 Best Paper
Mungmeeprued, Thisanaporn; Ma, Yuxin; Mehta, Nisarg; Lipani, Aldo
In: Proceedings of the ACM Symposium on Document Engineering, ACM, San Jose, CA, USA, 2022.
@inproceedings{mungmeeprued-etal-2022-tab-this-best,
title = {Tab this Folder of Documents: Page Stream Segmentation of Business Documents},
author = {Thisanaporn Mungmeeprued and Yuxin Ma and Nisarg Mehta and Aldo Lipani},
url = {https://www.researchgate.net/publication/363113372_Tab_this_Folder_of_Documents_Page_Stream_Segmentation_of_Business_Documents},
year = {2022},
date = {2022-09-20},
urldate = {2022-09-20},
booktitle = {Proceedings of the ACM Symposium on Document Engineering},
publisher = {ACM},
address = {San Jose, CA, USA},
series = {DocEng '22},
abstract = {In the midst of digital transformation, automatically understanding the structure and composition of scanned documents is important in order to allow correct indexing, archiving, and processing. In many organizations, different types of documents are usually scanned together in folders, so it is essential to automate the task of segmenting the folders into documents which then proceed to further analysis tailored to specific document types. This task is known as Page Stream Segmentation (PSS). In this paper, we propose a deep learning solution to solve the task of determining whether or not a page is a breaking-point given a sequence of scanned pages (a folder) as input. We also provide a dataset called TABME (TAB this folder of docuMEnts) generated specifically for this task. Our proposed architecture combines LayoutLM and ResNet to exploit both textual and visual features of the document pages and achieves an F1 score of 0.953. The dataset and code used to run the experiments in this paper are available at the following web link: https://github.com/aldolipani/TABME.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
In the midst of digital transformation, automatically understanding the structure and composition of scanned documents is important in order to allow correct indexing, archiving, and processing. In many organizations, different types of documents are usually scanned together in folders, so it is essential to automate the task of segmenting the folders into documents which then proceed to further analysis tailored to specific document types. This task is known as Page Stream Segmentation (PSS). In this paper, we propose a deep learning solution to solve the task of determining whether or not a page is a breaking-point given a sequence of scanned pages (a folder) as input. We also provide a dataset called TABME (TAB this folder of docuMEnts) generated specifically for this task. Our proposed architecture combines LayoutLM and ResNet to exploit both textual and visual features of the document pages and achieves an F1 score of 0.953. The dataset and code used to run the experiments in this paper are available at the following web link: https://github.com/aldolipani/TABME.
Evaluating the Cranfield Paradigm for Conversational Search Systems Inproceedings 🏆 Best Paper
Fu, Xiao; Yilmaz, Emine; Lipani, Aldo
In: Proceedings of the 12th International Conference on The Theory of Information Retrieval, ACM, Madrid, Spain, 2022.
@inproceedings{fu-etal-2022-evaluating-best,
title = {Evaluating the Cranfield Paradigm for Conversational Search Systems},
author = {Xiao Fu and Emine Yilmaz and Aldo Lipani},
doi = {10.1145/3539813.3545126},
year = {2022},
date = {2022-07-11},
urldate = {2022-01-01},
booktitle = {Proceedings of the 12th International Conference on The Theory of Information Retrieval},
publisher = {ACM},
address = {Madrid, Spain},
series = {ICTIR '22},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
A Multi-Task Based Neural Model to Simulate Users in Goal-Oriented Dialogue Systems Inproceedings
Kim, To Eun; Lipani, Aldo
In: Proc. of SIGIR, 2022.
@inproceedings{kim-lipani-2022-multi,
title = {A Multi-Task Based Neural Model to Simulate Users in Goal-Oriented Dialogue Systems},
author = {To Eun Kim and Aldo Lipani},
url = {https://www.researchgate.net/publication/360276605_A_Multi-Task_Based_Neural_Model_to_Simulate_Users_in_Goal-Oriented_Dialogue_Systems},
year = {2022},
date = {2022-06-11},
urldate = {2022-06-11},
booktitle = {Proc.~of SIGIR},
series = {SIGIR '22},
abstract = {A human-like user simulator that anticipates users' satisfaction scores, actions, and utterances can help goal-oriented dialogue systems in evaluating the conversation and refining their dialogue strategies. However, little work has experimented with user simulators which can generate users' utterances. In this paper, we propose a deep learning-based user simulator that predicts users' satisfaction scores and actions while also jointly generating users' utterances in a multi-task manner. In particular, we show that 1) the proposed deep text-to-text multi-task neural model achieves state-of-the-art performance in the users' satisfaction scores and actions prediction tasks, and 2) in an ablation analysis, user satisfaction score prediction, action prediction, and utterance generation tasks can boost the performance with each other via positive transfers across the tasks. The source code and model checkpoints used for the experiments run in this paper are available at the following weblink: https://github.com/kimdanny/user-simulation-t5.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
A human-like user simulator that anticipates users' satisfaction scores, actions, and utterances can help goal-oriented dialogue systems in evaluating the conversation and refining their dialogue strategies. However, little work has experimented with user simulators which can generate users' utterances. In this paper, we propose a deep learning-based user simulator that predicts users' satisfaction scores and actions while also jointly generating users' utterances in a multi-task manner. In particular, we show that 1) the proposed deep text-to-text multi-task neural model achieves state-of-the-art performance in the users' satisfaction scores and actions prediction tasks, and 2) in an ablation analysis, user satisfaction score prediction, action prediction, and utterance generation tasks can boost the performance with each other via positive transfers across the tasks. The source code and model checkpoints used for the experiments run in this paper are available at the following weblink: https://github.com/kimdanny/user-simulation-t5.
StepGame: A New Benchmark for Robust Multi-Hop Spatial Reasoning in Texts Inproceedings
Shi, Zhengxiang; Zhang, Qiang; Lipani, Aldo
In: Proceedings of the Association for the Advancement of Artificial Intelligence, 2022.
@inproceedings{Shi2022,
title = {StepGame: A New Benchmark for Robust Multi-Hop Spatial Reasoning in Texts},
author = {Zhengxiang Shi and Qiang Zhang and Aldo Lipani},
url = {https://www.researchgate.net/publication/357159030_StepGame_A_New_Benchmark_for_Robust_Multi-Hop_Spatial_Reasoning_in_Texts},
year = {2022},
date = {2022-01-01},
booktitle = {Proceedings of the Association for the Advancement of Artificial Intelligence},
series = {AAAI '22},
abstract = {Inferring spatial relations in natural language is a crucial ability an intelligent system should possess. The bAbI dataset tries to capture tasks relevant to this domain (tasks 17 and 19). However, these tasks have several limitations. Most importantly, they are limited to fixed expressions, they are limited in the number of reasoning steps required to solve them, and they fail to test the robustness of models to input that contains irrelevant or redundant information. In this paper, we present a new Question-Answering dataset called StepGame for robust multi-hop spatial reasoning in texts. Our experiments demonstrate that state-of-the-art models on the bAbI dataset struggle on the StepGame dataset. Moreover, we propose a Tensor-Product based Memory-Augmented Neural Network (TP-MANN) specialized for spatial reasoning tasks. Experimental results on both datasets show that our model outperforms all the baselines with superior generalization and robustness performance.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Inferring spatial relations in natural language is a crucial ability an intelligent system should possess. The bAbI dataset tries to capture tasks relevant to this domain (tasks 17 and 19). However, these tasks have several limitations. Most importantly, they are limited to fixed expressions, they are limited in the number of reasoning steps required to solve them, and they fail to test the robustness of models to input that contains irrelevant or redundant information. In this paper, we present a new Question-Answering dataset called StepGame for robust multi-hop spatial reasoning in texts. Our experiments demonstrate that state-of-the-art models on the bAbI dataset struggle on the StepGame dataset. Moreover, we propose a Tensor-Product based Memory-Augmented Neural Network (TP-MANN) specialized for spatial reasoning tasks. Experimental results on both datasets show that our model outperforms all the baselines with superior generalization and robustness performance.
Forecasting Solar Home System Customers' Electricity Usage with a 3D Convolutional Neural Network to Improve Energy Access Journal Article
Kizilcec, Vivien; Spataru, Catalina; Lipani, Aldo; Parikh, Priti
In: Energies, vol. 15, no. 3, 2022, ISSN: 1996-1073.
@article{en15030857,
title = {Forecasting Solar Home System Customers' Electricity Usage with a 3D Convolutional Neural Network to Improve Energy Access},
author = {Vivien Kizilcec and Catalina Spataru and Aldo Lipani and Priti Parikh},
url = {https://www.mdpi.com/1996-1073/15/3/857},
doi = {10.3390/en15030857},
issn = {1996-1073},
year = {2022},
date = {2022-01-01},
urldate = {2022-01-01},
journal = {Energies},
volume = {15},
number = {3},
abstract = {Off-grid technologies, such as solar home systems (SHS), offer the opportunity to alleviate global energy poverty, providing a cost-effective alternative to an electricity grid connection. However, there is a paucity of high-quality SHS electricity usage data and thus a limited understanding of consumers’ past and future usage patterns. This study addresses this gap by providing a rare large-scale analysis of real-time energy consumption data for SHS customers (n = 63,299) in Rwanda. Our results show that 70% of SHS users’ electricity usage decreased a year after their SHS was installed. This paper is novel in its application of a three-dimensional convolutional neural network (CNN) architecture for electricity load forecasting using time series data. It also marks the first time a CNN was used to predict SHS customers’ electricity consumption. The model forecasts individual households’ usage 24 h and seven days ahead, as well as an average week across the next three months. The last scenario derived the best performance with a mean squared error of 0.369. SHS companies could use these predictions to offer a tailored service to customers, including providing feedback information on their likely future usage and expenditure. The CNN could also aid load balancing for SHS based microgrids.},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
Off-grid technologies, such as solar home systems (SHS), offer the opportunity to alleviate global energy poverty, providing a cost-effective alternative to an electricity grid connection. However, there is a paucity of high-quality SHS electricity usage data and thus a limited understanding of consumers’ past and future usage patterns. This study addresses this gap by providing a rare large-scale analysis of real-time energy consumption data for SHS customers (n = 63,299) in Rwanda. Our results show that 70% of SHS users’ electricity usage decreased a year after their SHS was installed. This paper is novel in its application of a three-dimensional convolutional neural network (CNN) architecture for electricity load forecasting using time series data. It also marks the first time a CNN was used to predict SHS customers’ electricity consumption. The model forecasts individual households’ usage 24 h and seven days ahead, as well as an average week across the next three months. The last scenario derived the best performance with a mean squared error of 0.369. SHS companies could use these predictions to offer a tailored service to customers, including providing feedback information on their likely future usage and expenditure. The CNN could also aid load balancing for SHS based microgrids.
Semantic segmentation of cracks: Data challenges and architecture Journal Article
Panella, Fabio; Lipani, Aldo; Boehm, Jan
In: Automation in Construction, vol. 135, pp. 104110, 2022, ISSN: 0926-5805.
@article{PANELLA2022104110,
title = {Semantic segmentation of cracks: Data challenges and architecture},
author = {Fabio Panella and Aldo Lipani and Jan Boehm},
url = {https://www.sciencedirect.com/science/article/pii/S0926580521005616},
doi = {https://doi.org/10.1016/j.autcon.2021.104110},
issn = {0926-5805},
year = {2022},
date = {2022-01-01},
journal = {Automation in Construction},
volume = {135},
pages = {104110},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
Borough-level COVID-19 forecasting in London using deep learning techniques and a novel MSE-Moran’s I loss function Journal Article
Olsen, Frederik; Schillaci, Calogero; Ibrahim, Mohamed; Lipani, Aldo
In: Results in Physics, vol. 35, pp. 105374, 2022, ISSN: 2211-3797.
@article{OLSEN2022105374,
title = {Borough-level COVID-19 forecasting in London using deep learning techniques and a novel MSE-Moran’s I loss function},
author = {Frederik Olsen and Calogero Schillaci and Mohamed Ibrahim and Aldo Lipani},
url = {https://www.sciencedirect.com/science/article/pii/S2211379722001450},
doi = {https://doi.org/10.1016/j.rinp.2022.105374},
issn = {2211-3797},
year = {2022},
date = {2022-01-01},
journal = {Results in Physics},
volume = {35},
pages = {105374},
abstract = {Following its identification in late 2019, COVID-19 has spread around the globe, and been declared a pandemic. With this in mind, modelling the spread of COVID-19 remains important for responding effectively. To date research has focused primarily on modelling the spread of COVID-19 on national and regional scales with just a few studies doing so on a city and sub-city scale. However, no attempts have yet been made to design and optimize a model explicitly for accurately forecasting the spread of COVID-19 at sub-city scale. This research aimed to address this research gap by developing an experimental LSTM-ANN deep learning model. The model is largely autoregressive in nature as it considers temporally lagged borough-level COVID-19 cases data from the last 9 days, but also considers temporally lagged (i) borough-level NO2 concentration data, (ii) government stringency data, and (iii) climatic data from the last 9 days, as well as non-temporally variable borough-level urban characteristics data when modelling and forecasting the spread of the disease. The model was also encouraged to learn the spatial relationships between boroughs with regards to the spread of COVID-19 by a novel MSE-Moran’s I loss function. Overall, the model’s performance appears promising and so the model represents a useful tool for assisting the decision making and interventions of governing bodies within cities. A sensitivity analysis also indicated that of the non COVID-19 variables, the government stringency is particularly important in the modelling process, with this being closely followed by the climatic variables, the NO2 concentration data, and finally the urban characteristics data. Additionally, the introduction of the novel MSE-Moran’s I loss function appeared to improve the model’s forecasting performance, and so this research has implications at the intersection of deep learning and disease modelling. It may also have implications within spatio-temporal forecasting more generally because such a feature may have the potential to improve forecasting in other spatio-temporal applications},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
Following its identification in late 2019, COVID-19 has spread around the globe, and been declared a pandemic. With this in mind, modelling the spread of COVID-19 remains important for responding effectively. To date research has focused primarily on modelling the spread of COVID-19 on national and regional scales with just a few studies doing so on a city and sub-city scale. However, no attempts have yet been made to design and optimize a model explicitly for accurately forecasting the spread of COVID-19 at sub-city scale. This research aimed to address this research gap by developing an experimental LSTM-ANN deep learning model. The model is largely autoregressive in nature as it considers temporally lagged borough-level COVID-19 cases data from the last 9 days, but also considers temporally lagged (i) borough-level NO2 concentration data, (ii) government stringency data, and (iii) climatic data from the last 9 days, as well as non-temporally variable borough-level urban characteristics data when modelling and forecasting the spread of the disease. The model was also encouraged to learn the spatial relationships between boroughs with regards to the spread of COVID-19 by a novel MSE-Moran’s I loss function. Overall, the model’s performance appears promising and so the model represents a useful tool for assisting the decision making and interventions of governing bodies within cities. A sensitivity analysis also indicated that of the non COVID-19 variables, the government stringency is particularly important in the modelling process, with this being closely followed by the climatic variables, the NO2 concentration data, and finally the urban characteristics data. Additionally, the introduction of the novel MSE-Moran’s I loss function appeared to improve the model’s forecasting performance, and so this research has implications at the intersection of deep learning and disease modelling. It may also have implications within spatio-temporal forecasting more generally because such a feature may have the potential to improve forecasting in other spatio-temporal applications
On The Prediction of Landslide Occurrences and Sizesvia Hierarchical Neural Networks Journal Article
Aguilera, Quinton; Lombardo, Luigi; Tanyas, Hakan; Lipani, Aldo
In: Stochastic Environmental Research and Risk Assessment, 2022, ISSN: 2693-5015.
@article{Aguilera2022,
title = {On The Prediction of Landslide Occurrences and Sizesvia Hierarchical Neural Networks},
author = {Quinton Aguilera and Luigi Lombardo and Hakan Tanyas and Aldo Lipani},
url = {https://doi.org/10.21203/rs.3.rs-1260650/v1},
doi = {10.21203/rs.3.rs-1260650/v1},
issn = {2693-5015},
year = {2022},
date = {2022-01-01},
journal = {Stochastic Environmental Research and Risk Assessment},
abstract = {For more than three decades, the scientific community that studies landslides through data-driven models has focused on estimating where landslides occur across a given landscape. This concept is widely known as landslide susceptibility. And, it has seen a vast improvement from old bivariate statistical techniques to modern deep learning routines. Despite all these advancements, no spatially-explicit data-driven model is currently capable of also predicting how large landslides may be once they trigger in a specific study area. In this work, we exploit a model architecture that has already found a number of applications in landslide susceptibility. Specifically, we opt for the use of Neural Network (NN). But, instead of focusing exclusively on where landslides may occur, we extend this paradigm to also spatially predict classes of landslide sizes. As a result, we keep the traditional binary classification paradigm but we make use of it to complement the susceptibility estimates with a crucial information for landslide hazard assessment. We will refer to this model as Hierarchical Neural Network (HNN) throughout the manuscript. To test this analytical protocol, we use the Nepalese area where the Gorkha earthquake induced tens of thousands of landslides in 2014. The results we obtain are quite promising. The component of our HNN that estimates the susceptibility outperforms a binomial Generalized Linear Model (GLM) baseline we used as benchmark. We did this for a GLM represents the most common classifier in the landslide literature. Most importantly, our HNN also suitably performed across the entire procedure. As a result, the landslide-area-class prediction returned not just a single susceptibility map, as per tradition. But, it also produced several informative maps on the expected landslide size classes. Our vision is for administrations to consult these suite of model outputs and maps to better assess the risk to local communities and infrastructure. And, to promote the diffusion of our HNN, we are sharing the data and codes in the supplementary material in the hope that we would stimulate others to replicate similar analyses.},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
For more than three decades, the scientific community that studies landslides through data-driven models has focused on estimating where landslides occur across a given landscape. This concept is widely known as landslide susceptibility. And, it has seen a vast improvement from old bivariate statistical techniques to modern deep learning routines. Despite all these advancements, no spatially-explicit data-driven model is currently capable of also predicting how large landslides may be once they trigger in a specific study area. In this work, we exploit a model architecture that has already found a number of applications in landslide susceptibility. Specifically, we opt for the use of Neural Network (NN). But, instead of focusing exclusively on where landslides may occur, we extend this paradigm to also spatially predict classes of landslide sizes. As a result, we keep the traditional binary classification paradigm but we make use of it to complement the susceptibility estimates with a crucial information for landslide hazard assessment. We will refer to this model as Hierarchical Neural Network (HNN) throughout the manuscript. To test this analytical protocol, we use the Nepalese area where the Gorkha earthquake induced tens of thousands of landslides in 2014. The results we obtain are quite promising. The component of our HNN that estimates the susceptibility outperforms a binomial Generalized Linear Model (GLM) baseline we used as benchmark. We did this for a GLM represents the most common classifier in the landslide literature. Most importantly, our HNN also suitably performed across the entire procedure. As a result, the landslide-area-class prediction returned not just a single susceptibility map, as per tradition. But, it also produced several informative maps on the expected landslide size classes. Our vision is for administrations to consult these suite of model outputs and maps to better assess the risk to local communities and infrastructure. And, to promote the diffusion of our HNN, we are sharing the data and codes in the supplementary material in the hope that we would stimulate others to replicate similar analyses.
Dynamic Schema Graph Fusion Network for Multi-Domain Dialogue State Tracking Inproceedings
Feng, Yue; Lipani, Aldo; Ye, Fanghua; Zhang, Qiang; Yilmaz, Emine
In: Association for Computational Linguistics: ACL 2022, Association for Computational Linguistics, 2022.
@inproceedings{feng-etal-2022-dynamic,
title = {Dynamic Schema Graph Fusion Network for Multi-Domain Dialogue State Tracking},
author = {Yue Feng and Aldo Lipani and Fanghua Ye and Qiang Zhang and Emine Yilmaz},
url = {https://aclanthology.org/2022.acl-long.10.pdf},
year = {2022},
date = {2022-01-01},
urldate = {2022-01-01},
booktitle = {Association for Computational Linguistics: ACL 2022},
publisher = {Association for Computational Linguistics},
abstract = {Dialogue State Tracking (DST) aims to keep track of users' intentions during the course of a conversation. In DST, modelling the relations among domains and slots is still an under-studied problem. Existing approaches that have considered such relations generally fall short in: (1) fusing prior slot-domain membership relations and dialogue-aware dynamic slot relations explicitly, and (2) generalizing to unseen domains. To address these issues, we propose a novel textbfDynamic textbfSchema textbfGraph textbfFusion textbfNetwork (textbfDSGFNet), which generates a dynamic schema graph to explicitly fuse the prior slot-domain membership relations and dialogue-aware dynamic slot relations. It also uses the schemata to facilitate knowledge transfer to new domains. DSGFNet consists of a dialogue utterance encoder, a schema graph encoder, a dialogue-aware schema graph evolving network, and a schema graph enhanced dialogue state decoder. Empirical results on benchmark datasets (i.e., SGD, MultiWOZ2.1, and MultiWOZ2.2), show that DSGFNet outperforms existing methods.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Dialogue State Tracking (DST) aims to keep track of users' intentions during the course of a conversation. In DST, modelling the relations among domains and slots is still an under-studied problem. Existing approaches that have considered such relations generally fall short in: (1) fusing prior slot-domain membership relations and dialogue-aware dynamic slot relations explicitly, and (2) generalizing to unseen domains. To address these issues, we propose a novel textbfDynamic textbfSchema textbfGraph textbfFusion textbfNetwork (textbfDSGFNet), which generates a dynamic schema graph to explicitly fuse the prior slot-domain membership relations and dialogue-aware dynamic slot relations. It also uses the schemata to facilitate knowledge transfer to new domains. DSGFNet consists of a dialogue utterance encoder, a schema graph encoder, a dialogue-aware schema graph evolving network, and a schema graph enhanced dialogue state decoder. Empirical results on benchmark datasets (i.e., SGD, MultiWOZ2.1, and MultiWOZ2.2), show that DSGFNet outperforms existing methods.
Learning to Execute Actions or Ask Clarification Questions Inproceedings
Shi, Zhengxiang; Feng, Yue; Lipani, Aldo
In: Findings of NAACL, 2022.
@inproceedings{shi-etal-2022-learning,
title = {Learning to Execute Actions or Ask Clarification Questions},
author = {Zhengxiang Shi and Yue Feng and Aldo Lipani},
url = {https://www.researchgate.net/publication/360050130_Learning_to_Execute_Actions_or_Ask_Clarification_Questions},
year = {2022},
date = {2022-01-01},
urldate = {2022-01-01},
booktitle = {Findings of NAACL},
series = {NAACL '22},
abstract = {Collaborative tasks are ubiquitous activities where a form of communication is required in order to reach a joint goal. Collaborative building is one of such tasks. We wish to develop an intelligent builder agent in a simulated building environment (Minecraft) that can build whatever users wish to build by just talking to the agent. In order to achieve this goal, such agents need to be able to take the initiative by asking clarification questions when further information is needed. Existing works on Minecraft Corpus Dataset only learn to execute instructions neglecting the importance of asking for clarifications. In this paper, we extend the Minecraft Corpus Dataset by annotating all builder utterances into eight types, including clarification questions, and propose a new builder agent model capable of determining when to ask or execute instructions. Experimental results show that our model achieves state-of-the-art performance on the collabora-tive building task with a substantial improvement. We also define two new tasks, the learning to ask task and the joint learning task. The latter consists of solving both collaborating building and learning to ask tasks jointly.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Collaborative tasks are ubiquitous activities where a form of communication is required in order to reach a joint goal. Collaborative building is one of such tasks. We wish to develop an intelligent builder agent in a simulated building environment (Minecraft) that can build whatever users wish to build by just talking to the agent. In order to achieve this goal, such agents need to be able to take the initiative by asking clarification questions when further information is needed. Existing works on Minecraft Corpus Dataset only learn to execute instructions neglecting the importance of asking for clarifications. In this paper, we extend the Minecraft Corpus Dataset by annotating all builder utterances into eight types, including clarification questions, and propose a new builder agent model capable of determining when to ask or execute instructions. Experimental results show that our model achieves state-of-the-art performance on the collabora-tive building task with a substantial improvement. We also define two new tasks, the learning to ask task and the joint learning task. The latter consists of solving both collaborating building and learning to ask tasks jointly.
Attention-based Ingredient Phrase Parser Inproceedings
Shi, Zhengxiang; Ni, Pin; Kim, To Eun; Wang, Mehui; Lipani, Aldo
In: 2022.
@inproceedings{Shi2022:Ingredient,
title = {Attention-based Ingredient Phrase Parser},
author = {Zhengxiang Shi and Pin Ni and To Eun Kim and Mehui Wang and Aldo Lipani},
year = {2022},
date = {2022-01-01},
series = {ESANN},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Snow detection in alpine regions with Convolutional Neural Networks: discriminating snow from cold clouds and water body Journal Article
Lu, Yichen; James, Thomas; Schillaci, Calogero; Lipani, Aldo
In: GIScience & Remote Sensing, vol. 59, no. 1, pp. 1321-1343, 2022.
@article{Lu2022,
title = {Snow detection in alpine regions with Convolutional Neural Networks: discriminating snow from cold clouds and water body},
author = {Yichen Lu and Thomas James and Calogero Schillaci and Aldo Lipani},
url = {https://www.tandfonline.com/doi/abs/10.1080/15481603.2022.2112391},
doi = {10.1080/15481603.2022.2112391},
year = {2022},
date = {2022-01-01},
journal = {GIScience \& Remote Sensing},
volume = {59},
number = {1},
pages = {1321-1343},
publisher = {Taylor \& Francis},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
Eigenvector-based Graph Neural Network Embeddings and Trust Rating Prediction in Bitcoin Networks Inproceedings
Ni, Pin; Yuan, Qiao; Khraishi, Raad; Okhrati, Ramin; Lipani, Aldo; Medda, Francesca
In: Proceedings of the 3rd ACM International Conference on AI in Finance, ACM, 2022.
@inproceedings{pin-eta-al-2022-eigenvector-based,
title = {Eigenvector-based Graph Neural Network Embeddings and Trust Rating Prediction in Bitcoin Networks},
author = {Pin Ni and Qiao Yuan and Raad Khraishi and Ramin Okhrati and Aldo Lipani and Francesca Medda},
year = {2022},
date = {2022-01-01},
booktitle = {Proceedings of the 3rd ACM International Conference on AI in Finance},
publisher = {ACM},
series = {ICAIF '22},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Deep Learning Based Burnt Area Mapping Using Sentinel 1 for the Santa Cruz Mountains Lightning Complex (CZU) and Creek Fires 2020 Journal Article
Luft, Harrison; Schillaci, Calogero; Ceccherini, Guido; Vieira, Diana; Lipani, Aldo
In: Fire, vol. 5, no. 5, 2022, ISSN: 2571-6255.
@article{luft-etal-2022-deep-learning-based,
title = {Deep Learning Based Burnt Area Mapping Using Sentinel 1 for the Santa Cruz Mountains Lightning Complex (CZU) and Creek Fires 2020},
author = {Harrison Luft and Calogero Schillaci and Guido Ceccherini and Diana Vieira and Aldo Lipani},
url = {https://www.mdpi.com/2571-6255/5/5/163},
doi = {10.3390/fire5050163},
issn = {2571-6255},
year = {2022},
date = {2022-01-01},
urldate = {2022-01-01},
journal = {Fire},
volume = {5},
number = {5},
abstract = {The study presented here builds on previous synthetic aperture radar (SAR) burnt area estimation models and presents the first U-Net (a convolutional network architecture for fast and precise segmentation of images) combined with ResNet50 (Residual Networks used as a backbone for many computer vision tasks) encoder architecture used with SAR, Digital Elevation Model, and land cover data for burnt area mapping in near-real time. The Santa Cruz Mountains Lightning Complex (CZU) was one of the most destructive fires in state history. The results showed a maximum burnt area segmentation F1-Score of 0.671 in the CZU, which outperforms current models estimating burnt area with SAR data for the specific event studied models in the literature, with an F1-Score of 0.667. The framework presented here has the potential to be applied on a near real-time basis, which could allow land monitoring as the frequency of data capture improves.},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
The study presented here builds on previous synthetic aperture radar (SAR) burnt area estimation models and presents the first U-Net (a convolutional network architecture for fast and precise segmentation of images) combined with ResNet50 (Residual Networks used as a backbone for many computer vision tasks) encoder architecture used with SAR, Digital Elevation Model, and land cover data for burnt area mapping in near-real time. The Santa Cruz Mountains Lightning Complex (CZU) was one of the most destructive fires in state history. The results showed a maximum burnt area segmentation F1-Score of 0.671 in the CZU, which outperforms current models estimating burnt area with SAR data for the specific event studied models in the literature, with an F1-Score of 0.667. The framework presented here has the potential to be applied on a near real-time basis, which could allow land monitoring as the frequency of data capture improves.
Deep Learning Based Burnt Area Mapping Using Sentinel 1 for the Santa Cruz Mountains Lightning Complex (CZU) and Creek Fires 2020 Journal Article
Luft, Harrison; Schillaci, Calogero; Ceccherini, Guido; Vieira, Diana; Lipani, Aldo
In: Fire, vol. 5, no. 5, 2022, ISSN: 2571-6255.
@article{luft-etal-2022-deep-learning-based,
title = {Deep Learning Based Burnt Area Mapping Using Sentinel 1 for the Santa Cruz Mountains Lightning Complex (CZU) and Creek Fires 2020},
author = {Harrison Luft and Calogero Schillaci and Guido Ceccherini and Diana Vieira and Aldo Lipani},
url = {https://www.mdpi.com/2571-6255/5/5/163},
doi = {10.3390/fire5050163},
issn = {2571-6255},
year = {2022},
date = {2022-01-01},
journal = {Fire},
volume = {5},
number = {5},
abstract = {The study presented here builds on previous synthetic aperture radar (SAR) burnt area estimation models and presents the first U-Net (a convolutional network architecture for fast and precise segmentation of images) combined with ResNet50 (Residual Networks used as a backbone for many computer vision tasks) encoder architecture used with SAR, Digital Elevation Model, and land cover data for burnt area mapping in near-real time. The Santa Cruz Mountains Lightning Complex (CZU) was one of the most destructive fires in state history. The results showed a maximum burnt area segmentation F1-Score of 0.671 in the CZU, which outperforms current models estimating burnt area with SAR data for the specific event studied models in the literature, with an F1-Score of 0.667. The framework presented here has the potential to be applied on a near real-time basis, which could allow land monitoring as the frequency of data capture improves.},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
The study presented here builds on previous synthetic aperture radar (SAR) burnt area estimation models and presents the first U-Net (a convolutional network architecture for fast and precise segmentation of images) combined with ResNet50 (Residual Networks used as a backbone for many computer vision tasks) encoder architecture used with SAR, Digital Elevation Model, and land cover data for burnt area mapping in near-real time. The Santa Cruz Mountains Lightning Complex (CZU) was one of the most destructive fires in state history. The results showed a maximum burnt area segmentation F1-Score of 0.671 in the CZU, which outperforms current models estimating burnt area with SAR data for the specific event studied models in the literature, with an F1-Score of 0.667. The framework presented here has the potential to be applied on a near real-time basis, which could allow land monitoring as the frequency of data capture improves.
2021
Schillaci, Calogero; Saia, Sergio; Lipani, Aldo; Perego, Alessia; Zaccone, Claudio; Acutis, Marco
In: Carbon Balance and Management 2021 16:1, vol. 16, no. 1, pp. 1–15, 2021, ISSN: 1750-0680.
@article{Schillaci2021,
title = {Validating the regional estimates of changes in soil organic carbon by using the data from paired-sites: the case study of Mediterranean arable lands},
author = {Calogero Schillaci and Sergio Saia and Aldo Lipani and Alessia Perego and Claudio Zaccone and Marco Acutis},
url = {https://cbmjournal.biomedcentral.com/articles/10.1186/s13021-021-00182-7},
doi = {10.1186/S13021-021-00182-7},
issn = {1750-0680},
year = {2021},
date = {2021-06-01},
journal = {Carbon Balance and Management 2021 16:1},
volume = {16},
number = {1},
pages = {1--15},
publisher = {BioMed Central},
abstract = {Legacy data are unique occasions for estimating soil organic carbon (SOC) concentration changes and spatial variability, but their use showed limitations due to the sampling schemes adopted and improvements may be needed in the analysis methodologies. When SOC changes is estimated with legacy data, the use of soil samples collected in different plots (i.e., non-paired data) may lead to biased results. In the present work},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
Legacy data are unique occasions for estimating soil organic carbon (SOC) concentration changes and spatial variability, but their use showed limitations due to the sampling schemes adopted and improvements may be needed in the analysis methodologies. When SOC changes is estimated with legacy data, the use of soil samples collected in different plots (i.e., non-paired data) may lead to biased results. In the present work
Mitigating the Position Bias of Transformer Models in Passage Re-Ranking Inproceedings
Hofstätter, Sebastian; Lipani, Aldo; Althammer, Sophia; Zlabinger, Markus; Hanbury, Allan
In: 2021.
@inproceedings{hofstaetter21,
title = {Mitigating the Position Bias of Transformer Models in Passage Re-Ranking},
author = {Sebastian Hofst\"{a}tter and Aldo Lipani and Sophia Althammer and Markus Zlabinger and Allan Hanbury},
url = {https://www.researchgate.net/publication/348589683_Mitigating_the_Position_Bias_of_Transformer_Models_in_Passage_Re-Ranking},
year = {2021},
date = {2021-03-28},
series = {ECIR},
abstract = {Supervised machine learning models and their evaluation strongly depends on the quality of the underlying dataset. When we search for a relevant piece of information it may appear anywhere in a given passage. However, we observe a bias in the position of the correct answer in the text in two popular Question Answering datasets used for passage re-ranking. The excessive favoring of earlier positions inside passages is an unwanted artefact. This leads to three common Transformer-based re-ranking models to ignore relevant parts in unseen passages. More concerningly, as the evaluation set is taken from the same biased distribution, the models overfitting to that bias overestimate their true effectiveness. In this work we analyze position bias on datasets, the contextualized representations, and their effect on retrieval results. We propose a debiasing method for retrieval datasets. Our results show that a model trained on a position-biased dataset exhibits a significant decrease in re-ranking effectiveness when evaluated on a debiased dataset. We demonstrate that by mitigating the position bias, Transformer-based re-ranking models are equally effective on a biased and debiased dataset, as well as more effective in a transfer-learning setting between two differently biased datasets.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Supervised machine learning models and their evaluation strongly depends on the quality of the underlying dataset. When we search for a relevant piece of information it may appear anywhere in a given passage. However, we observe a bias in the position of the correct answer in the text in two popular Question Answering datasets used for passage re-ranking. The excessive favoring of earlier positions inside passages is an unwanted artefact. This leads to three common Transformer-based re-ranking models to ignore relevant parts in unseen passages. More concerningly, as the evaluation set is taken from the same biased distribution, the models overfitting to that bias overestimate their true effectiveness. In this work we analyze position bias on datasets, the contextualized representations, and their effect on retrieval results. We propose a debiasing method for retrieval datasets. Our results show that a model trained on a position-biased dataset exhibits a significant decrease in re-ranking effectiveness when evaluated on a debiased dataset. We demonstrate that by mitigating the position bias, Transformer-based re-ranking models are equally effective on a biased and debiased dataset, as well as more effective in a transfer-learning setting between two differently biased datasets.
Predicting non-residential building fire risk using geospatial information and convolutional neural networks Journal Article
Anderson-Bell, Jake; Schillaci, Calogero; Lipani, Aldo
In: Remote Sensing Applications: Society and Environment, vol. 21, pp. 100470, 2021, ISBN: 2352-9385.
@article{AndersonBell2021,
title = {Predicting non-residential building fire risk using geospatial information and convolutional neural networks},
author = {Jake Anderson-Bell and Calogero Schillaci and Aldo Lipani},
url = {https://www.researchgate.net/publication/348685058_Predicting_Non-Residential_Building_Fire_Risk_using_geospatial_information_and_Convolutional_Neural_Networks?_sg=0qL6N0cbQ_HdrMLbhX3okclPN2GEgUaFvVGkR7KcWmzNXnHYLEGXGZy_MshQG22n4HcmLISR8yJciXOD4nvqHm6JuZzxot56EKlMkrp1.2swBtS-hEil0L0vmFPQpFL8YYSXszHMWp9gGihU0nLKrJpCVp-Bsc-hMwRS0koLkkPn-iFbc_J58y9Jgbkj73A},
doi = {10.1016/j.rsase.2021.100470},
isbn = {2352-9385},
year = {2021},
date = {2021-01-01},
journal = {Remote Sensing Applications: Society and Environment},
volume = {21},
pages = {100470},
abstract = {Building fire risk prediction is crucial for allocation of building inspection resources and prevention of fire incidents. Existing research of building fire prediction makes use of data relating to local demography, crime, building use and physical building characteristics, yet few studies have analysed the relative importance of predictive features. Furthermore, image features relating to buildings, such as aerial imagery and digital surface models (DSM), have not been explored. This research presents a multi-modal hybrid neural network for the prediction of fire risk at the building level using the London Fire Brigade dataset. The inclusion of traditional and novel image features is assessed using Shapley values and an ablation study. The ablation study found that while building use is the most effective contributor of classification performance, demographic features, apart from social class, are detrimental. Moreover, while the DSM did not lead to any notable improvement in classification performance, the inclusion of the aerial imagery feature lead to a 4% increase in median validation ROC AUC. The final model presented achieved an ROC AUC of 0.8195 on the test set.},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
Building fire risk prediction is crucial for allocation of building inspection resources and prevention of fire incidents. Existing research of building fire prediction makes use of data relating to local demography, crime, building use and physical building characteristics, yet few studies have analysed the relative importance of predictive features. Furthermore, image features relating to buildings, such as aerial imagery and digital surface models (DSM), have not been explored. This research presents a multi-modal hybrid neural network for the prediction of fire risk at the building level using the London Fire Brigade dataset. The inclusion of traditional and novel image features is assessed using Shapley values and an ablation study. The ablation study found that while building use is the most effective contributor of classification performance, demographic features, apart from social class, are detrimental. Moreover, while the DSM did not lead to any notable improvement in classification performance, the inclusion of the aerial imagery feature lead to a 4% increase in median validation ROC AUC. The final model presented achieved an ROC AUC of 0.8195 on the test set.
Evaluation metrics for measuring bias in search engine results Journal Article
Gezici, Gizem; Lipani, Aldo; Saygin, Yucel; Yilmaz, Emine
In: Information Retrieval Journal, pp. 1–29, 2021, ISSN: 1386-4564.
@article{Gezici2021,
title = {Evaluation metrics for measuring bias in search engine results},
author = {Gizem Gezici and Aldo Lipani and Yucel Saygin and Emine Yilmaz},
url = {http://link.springer.com/10.1007/s10791-020-09386-w},
doi = {10.1007/s10791-020-09386-w},
issn = {1386-4564},
year = {2021},
date = {2021-01-01},
journal = {Information Retrieval Journal},
pages = {1--29},
publisher = {Springer},
abstract = {Search engines decide what we see for a given search query. Since many people are exposed to information through search engines, it is fair to expect that search engines are neutral. However, search engine results do not necessarily cover all the viewpoints of a search query topic, and they can be biased towards a specific view since search engine results are returned based on relevance, which is calculated using many features and sophisticated algorithms where search neutrality is not necessarily the focal point. Therefore, it is important to evaluate the search engine results with respect to bias. In this work we propose novel web search bias evaluation measures which take into account the rank and relevance. We also propose a framework to evaluate web search bias using the proposed measures and test our framework on two popular search engines based on 57 controversial query topics such as abortion, medical marijuana, and gay marriage. We measure the stance bias (in support or against), as well as the ideological bias (conservative or liberal). We observe that the stance does not necessarily correlate with the ideological leaning, e.g. a positive stance on abortion indicates a liberal leaning but a positive stance on Cuba embargo indicates a conservative leaning. Our experiments show that neither of the search engines suffers from stance bias. However, both search engines suffer from ideological bias, both favouring one ideological leaning to the other, which is more significant from the perspective of polarisation in our society.},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
Search engines decide what we see for a given search query. Since many people are exposed to information through search engines, it is fair to expect that search engines are neutral. However, search engine results do not necessarily cover all the viewpoints of a search query topic, and they can be biased towards a specific view since search engine results are returned based on relevance, which is calculated using many features and sophisticated algorithms where search neutrality is not necessarily the focal point. Therefore, it is important to evaluate the search engine results with respect to bias. In this work we propose novel web search bias evaluation measures which take into account the rank and relevance. We also propose a framework to evaluate web search bias using the proposed measures and test our framework on two popular search engines based on 57 controversial query topics such as abortion, medical marijuana, and gay marriage. We measure the stance bias (in support or against), as well as the ideological bias (conservative or liberal). We observe that the stance does not necessarily correlate with the ideological leaning, e.g. a positive stance on abortion indicates a liberal leaning but a positive stance on Cuba embargo indicates a conservative leaning. Our experiments show that neither of the search engines suffers from stance bias. However, both search engines suffer from ideological bias, both favouring one ideological leaning to the other, which is more significant from the perspective of polarisation in our society.
Variational-LSTM autoencoder to forecast the spread of coronavirus across the globe Journal Article
Ibrahim, Mohamed R; Haworth, James; Lipani, Aldo; Aslam, Nilufer; Cheng, Tao; Christie, Nicola
In: PLOS ONE, vol. 16, no. 1, pp. 1-22, 2021.
@article{10.1371/journal.pone.0246120,
title = {Variational-LSTM autoencoder to forecast the spread of coronavirus across the globe},
author = {Mohamed R Ibrahim and James Haworth and Aldo Lipani and Nilufer Aslam and Tao Cheng and Nicola Christie},
url = {https://doi.org/10.1371/journal.pone.0246120
https://www.researchgate.net/publication/348855260_Variational-LSTM_autoencoder_to_forecast_the_spread_of_coronavirus_across_the_globe},
doi = {10.1371/journal.pone.0246120},
year = {2021},
date = {2021-01-01},
journal = {PLOS ONE},
volume = {16},
number = {1},
pages = {1-22},
publisher = {Public Library of Science},
abstract = {Modelling the spread of coronavirus globally while learning trends at global and country levels remains crucial for tackling the pandemic. We introduce a novel variational-LSTM Autoencoder model to predict the spread of coronavirus for each country across the globe. This deep Spatio-temporal model does not only rely on historical data of the virus spread but also includes factors related to urban characteristics represented in locational and demographic data (such as population density, urban population, and fertility rate), an index that represents the governmental measures and response amid toward mitigating the outbreak (includes 13 measures such as: 1) school closing, 2) workplace closing, 3) cancelling public events, 4) close public transport, 5) public information campaigns, 6) restrictions on internal movements, 7) international travel controls, 8) fiscal measures, 9) monetary measures, 10) emergency investment in health care, 11) investment in vaccines, 12) virus testing framework, and 13) contact tracing). In addition, the introduced method learns to generate a graph to adjust the spatial dependences among different countries while forecasting the spread. We trained two models for short and long-term forecasts. The first one is trained to output one step in future with three previous timestamps of all features across the globe, whereas the second model is trained to output 10 steps in future. Overall, the trained models show high validation for forecasting the spread for each country for short and long-term forecasts, which makes the introduce method a useful tool to assist decision and policymaking for the different corners of the globe.},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
Modelling the spread of coronavirus globally while learning trends at global and country levels remains crucial for tackling the pandemic. We introduce a novel variational-LSTM Autoencoder model to predict the spread of coronavirus for each country across the globe. This deep Spatio-temporal model does not only rely on historical data of the virus spread but also includes factors related to urban characteristics represented in locational and demographic data (such as population density, urban population, and fertility rate), an index that represents the governmental measures and response amid toward mitigating the outbreak (includes 13 measures such as: 1) school closing, 2) workplace closing, 3) cancelling public events, 4) close public transport, 5) public information campaigns, 6) restrictions on internal movements, 7) international travel controls, 8) fiscal measures, 9) monetary measures, 10) emergency investment in health care, 11) investment in vaccines, 12) virus testing framework, and 13) contact tracing). In addition, the introduced method learns to generate a graph to adjust the spatial dependences among different countries while forecasting the spread. We trained two models for short and long-term forecasts. The first one is trained to output one step in future with three previous timestamps of all features across the globe, whereas the second model is trained to output 10 steps in future. Overall, the trained models show high validation for forecasting the spread for each country for short and long-term forecasts, which makes the introduce method a useful tool to assist decision and policymaking for the different corners of the globe.
Learning Neural Point Processes with Latent Graphs Inproceedings
Zhang, Qiang; Lipani, Aldo; Yilmaz, Emine
In: Proceedings of the World Wide Web Conference, 2021.
@inproceedings{Zhang2021,
title = {Learning Neural Point Processes with Latent Graphs},
author = {Qiang Zhang and Aldo Lipani and Emine Yilmaz},
url = {https://www.researchgate.net/publication/349380768_Learning_Neural_Point_Processes_with_Latent_Graphs},
doi = {https://doi.org/10.1145/3442381.3450135},
year = {2021},
date = {2021-01-01},
booktitle = {Proceedings of the World Wide Web Conference},
series = {WWW '21},
abstract = {Neural point processes (NPPs) employ neural networks to capture complicated dynamics of asynchronous event sequences. Existing NPPs feed all history events into neural networks, assuming that all event types contribute to the prediction of the target type. However, this assumption can be problematic because in reality some event types do not contribute to the predictions of another type. To correct this defect, we learn to omit those types of events that do not contribute to the prediction of one target type during the formulation of NPPs. Towards this end, we simultaneously consider the tasks of (1) finding event types that contribute to predictions of the target types and (2) learning a NPP model from event sequences. For the former, we formulate a latent graph, with event types being vertices and non-zero contributing relationships being directed edges; then we propose a probabilistic graph generator, from which we sample a latent graph. For the latter, the sampled graph can be readily used as a plug-in to modify an existing NPP model. Because these two tasks are nested, we propose to optimize the model parameters through bilevel programming, and develop an efficient solution based on truncated gradient back-propagation. Experimental results on both synthetic and real-world datasets show the improved performance against state-of-the-art baselines. This work removes disturbance of non-contributing event types with the aid of a validation procedure, similar to the practice to mitigate overfitting used when training machine learning models.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Neural point processes (NPPs) employ neural networks to capture complicated dynamics of asynchronous event sequences. Existing NPPs feed all history events into neural networks, assuming that all event types contribute to the prediction of the target type. However, this assumption can be problematic because in reality some event types do not contribute to the predictions of another type. To correct this defect, we learn to omit those types of events that do not contribute to the prediction of one target type during the formulation of NPPs. Towards this end, we simultaneously consider the tasks of (1) finding event types that contribute to predictions of the target types and (2) learning a NPP model from event sequences. For the former, we formulate a latent graph, with event types being vertices and non-zero contributing relationships being directed edges; then we propose a probabilistic graph generator, from which we sample a latent graph. For the latter, the sampled graph can be readily used as a plug-in to modify an existing NPP model. Because these two tasks are nested, we propose to optimize the model parameters through bilevel programming, and develop an efficient solution based on truncated gradient back-propagation. Experimental results on both synthetic and real-world datasets show the improved performance against state-of-the-art baselines. This work removes disturbance of non-contributing event types with the aid of a validation procedure, similar to the practice to mitigate overfitting used when training machine learning models.
New pedotransfer approaches to predict soil bulk density using WoSIS soil data and environmental covariates in Mediterranean agro-ecosystems Journal Article
Schillaci, Calogero; Perego, Alessia; Valkama, Elena; Märker, Michael; Saia, Sergio; Veronesi, Fabio; Lipani, Aldo; Lombardo, Luigi; Tadiello, Tommaso; Gamper, Hannes A; Tedone, Luigi; Moss, Cami; Pareja-Serrano, Elena; Amato, Gabriele; Kühl, Kersten; Dămătîrcă, Claudia; Cogato, Alessia; Mzid, Nada; Eeswaran, Rasu; Rabelo, Marya; Sperandio, Giorgio; Bosino, Alberto; Bufalini, Margherita; Tunçay, Tülay; Ding, Jianqi; Fiorentini, Marco; Tiscornia, Guadalupe; Conradt, Sarah; Botta, Marco; Acutis, Marco
In: Science of The Total Environment, vol. 780, pp. 146609, 2021, ISSN: 0048-9697.
@article{SCHILLACI2021146609,
title = {New pedotransfer approaches to predict soil bulk density using WoSIS soil data and environmental covariates in Mediterranean agro-ecosystems},
author = {Calogero Schillaci and Alessia Perego and Elena Valkama and Michael M\"{a}rker and Sergio Saia and Fabio Veronesi and Aldo Lipani and Luigi Lombardo and Tommaso Tadiello and Hannes A Gamper and Luigi Tedone and Cami Moss and Elena Pareja-Serrano and Gabriele Amato and Kersten K\"{u}hl and Claudia D\u{a}m\u{a}t\^{i}rc\u{a} and Alessia Cogato and Nada Mzid and Rasu Eeswaran and Marya Rabelo and Giorgio Sperandio and Alberto Bosino and Margherita Bufalini and T\"{u}lay Tun\c{c}ay and Jianqi Ding and Marco Fiorentini and Guadalupe Tiscornia and Sarah Conradt and Marco Botta and Marco Acutis},
url = {https://www.sciencedirect.com/science/article/pii/S0048969721016776},
doi = {https://doi.org/10.1016/j.scitotenv.2021.146609},
issn = {0048-9697},
year = {2021},
date = {2021-01-01},
journal = {Science of The Total Environment},
volume = {780},
pages = {146609},
abstract = {For the estimation of the soil organic carbon stocks, bulk density (BD) is a fundamental parameter but measured data are usually not available especially when dealing with legacy soil data. It is possible to estimate BD by applying pedotransfer function (PTF). We applied different estimation methods with the aim to define a suitable PTF for BD of arable land for the Mediterranean Basin, which has peculiar climate features that may influence the soil carbon sequestration. To improve the existing BD estimation methods, we used a set of public climatic and topographic data along with the soil texture and organic carbon data. The present work consisted of the following steps: i) development of three PTFs models separately for top (0\textendash0.4 m) and subsoil (0.4\textendash1.2 m), ii) a 10-fold cross-validation, iii) model transferability using an external dataset derived from published data. The development of the new PTFs was based on the training dataset consisting of World Soil Information Service (WoSIS) soil profile data, climatic data from WorldClim at 1 km spatial resolution and Shuttle Radar Topography Mission (SRTM) digital elevation model at 30 m spatial resolution. The three PTFs models were developed using: Multiple Linear Regression stepwise (MLR-S), Multiple Linear Regression backward stepwise (MLR-BS), and Artificial Neural Network (ANN). The predictions of the newly developed PTFs were compared with the BD calculated using the PTF proposed by Manrique and Jones (MJ) and the modelled BD derived from the global SoilGrids dataset. For the topsoil training dataset (N = 129), MLR-S, MLR-BS and ANN had a R2 0.35, 0.58 and 0.86, respectively. For the model transferability, the three PTFs applied to the external topsoil dataset (N = 59), achieved R2 values of 0.06, 0.03 and 0.41. For the subsoil training dataset (N = 180), MLR-S, MLR-BS and ANN the R2 values were 0.36, 0.46 and 0.83, respectively. When applied to the external subsoil dataset (N = 29), the R2 values were 0.05, 0.06 and 0.41. The cross-validation for both top and subsoil dataset, resulted in an intermediate performance compared to calibration and validation with the external dataset. The new ANN PTF outperformed MLR-S, MLR-BS, MJ and SoilGrids approaches for estimating BD. Further improvements may be achieved by additionally considering the time of sampling, agricultural soil management and cultivation practices in predictive models.},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
For the estimation of the soil organic carbon stocks, bulk density (BD) is a fundamental parameter but measured data are usually not available especially when dealing with legacy soil data. It is possible to estimate BD by applying pedotransfer function (PTF). We applied different estimation methods with the aim to define a suitable PTF for BD of arable land for the Mediterranean Basin, which has peculiar climate features that may influence the soil carbon sequestration. To improve the existing BD estimation methods, we used a set of public climatic and topographic data along with the soil texture and organic carbon data. The present work consisted of the following steps: i) development of three PTFs models separately for top (0–0.4 m) and subsoil (0.4–1.2 m), ii) a 10-fold cross-validation, iii) model transferability using an external dataset derived from published data. The development of the new PTFs was based on the training dataset consisting of World Soil Information Service (WoSIS) soil profile data, climatic data from WorldClim at 1 km spatial resolution and Shuttle Radar Topography Mission (SRTM) digital elevation model at 30 m spatial resolution. The three PTFs models were developed using: Multiple Linear Regression stepwise (MLR-S), Multiple Linear Regression backward stepwise (MLR-BS), and Artificial Neural Network (ANN). The predictions of the newly developed PTFs were compared with the BD calculated using the PTF proposed by Manrique and Jones (MJ) and the modelled BD derived from the global SoilGrids dataset. For the topsoil training dataset (N = 129), MLR-S, MLR-BS and ANN had a R2 0.35, 0.58 and 0.86, respectively. For the model transferability, the three PTFs applied to the external topsoil dataset (N = 59), achieved R2 values of 0.06, 0.03 and 0.41. For the subsoil training dataset (N = 180), MLR-S, MLR-BS and ANN the R2 values were 0.36, 0.46 and 0.83, respectively. When applied to the external subsoil dataset (N = 29), the R2 values were 0.05, 0.06 and 0.41. The cross-validation for both top and subsoil dataset, resulted in an intermediate performance compared to calibration and validation with the external dataset. The new ANN PTF outperformed MLR-S, MLR-BS, MJ and SoilGrids approaches for estimating BD. Further improvements may be achieved by additionally considering the time of sampling, agricultural soil management and cultivation practices in predictive models.
How Am I Doing?: Evaluating Conversational Search Systems Offline Journal Article
Lipani, Aldo; Carterette, Ben; Yilmaz, Emine
In: ACM Transactions on Information Systems (TOIS), 2021.
@article{Lipani2021TOIS,
title = {How Am I Doing?: Evaluating Conversational Search Systems Offline},
author = {Aldo Lipani and Ben Carterette and Emine Yilmaz},
url = {https://www.researchgate.net/publication/350640565_How_Am_I_Doing_Evaluating_Conversational_Search_Systems_Offline
https://aldolipani.com/wp-content/uploads/2021/04/How_Am_I_Doing-Evaluating_Conversational_Search_Systems_Offline.pdf},
year = {2021},
date = {2021-01-01},
journal = {ACM Transactions on Information Systems (TOIS)},
abstract = {As conversational agents like Siri and Alexa gain in popularity and use, conversation is becoming a more and more important mode of interaction for search. Conversational search shares some features with traditional search, but differs in some important respects: conversational search systems are less likely to return ranked lists of results (a SERP), more likely to involve iterated interactions, and more likely to feature longer, well-formed user queries in the form of natural language questions. Because of these differences, traditional methods for search evaluation (such as the Cranfield paradigm) do not translate easily to conversational search. In this work, we propose a framework for offline evaluation of conversational search, which includes a methodology for creating test collections with relevance judgments, an evaluation measure based on a user interaction model, and an approach to collecting user interaction data to train the model. The framework is based on the idea of “subtopics”, often used to model novelty and diversity in search and recommendation, and the user model is similar to the geometric browsing model introduced by RBP and used in ERR. As far as we know, this is the first work to combine these ideas into a comprehensive framework for offline evaluation of conversational search.},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
As conversational agents like Siri and Alexa gain in popularity and use, conversation is becoming a more and more important mode of interaction for search. Conversational search shares some features with traditional search, but differs in some important respects: conversational search systems are less likely to return ranked lists of results (a SERP), more likely to involve iterated interactions, and more likely to feature longer, well-formed user queries in the form of natural language questions. Because of these differences, traditional methods for search evaluation (such as the Cranfield paradigm) do not translate easily to conversational search. In this work, we propose a framework for offline evaluation of conversational search, which includes a methodology for creating test collections with relevance judgments, an evaluation measure based on a user interaction model, and an approach to collecting user interaction data to train the model. The framework is based on the idea of “subtopics”, often used to model novelty and diversity in search and recommendation, and the user model is similar to the geometric browsing model introduced by RBP and used in ERR. As far as we know, this is the first work to combine these ideas into a comprehensive framework for offline evaluation of conversational search.
Convolutional neural networks for water segmentation using sentinel-2 red, green, blue (RGB) composites and derived spectral indices Journal Article
James, Thomas; Schillaci, Calogero; Lipani, Aldo
In: International Journal of Remote Sensing, vol. 42, no. 14, pp. 5342-5369, 2021.
@article{doi:10.1080/01431161.2021.1913298,
title = {Convolutional neural networks for water segmentation using sentinel-2 red, green, blue (RGB) composites and derived spectral indices},
author = {Thomas James and Calogero Schillaci and Aldo Lipani},
url = {https://doi.org/10.1080/01431161.2021.1913298
https://www.researchgate.net/publication/351093583_Convolutional_neural_networks_for_water_segmentation_using_sentinel-2_red_green_blue_RGB_composites_and_derived_spectral_indices},
doi = {10.1080/01431161.2021.1913298},
year = {2021},
date = {2021-01-01},
journal = {International Journal of Remote Sensing},
volume = {42},
number = {14},
pages = {5342-5369},
publisher = {Taylor \& Francis},
abstract = {Near-real time water segmentation with medium resolution satellite imagery plays a critical role in water management. Automated water segmentation of satellite imagery has traditionally been achieved using spectral indices. Spectral water segmentation is limited by environmental factors and requires human expertise to be applied effectively. In recent years, the use of convolutional neural networks (CNN’s) for water segmentation has been successful when used on high-resolution satellite imagery, but to a lesser extent for medium resolution imagery. Existing studies have been limited to geographically localized datasets and reported metrics have been benchmarked against a limited range of spectral indices. This study seeks to determine if a single CNN based on Red, Green, Blue (RGB) image classification can effectively segment water on a global scale and outperform traditional spectral methods. Additionally, this study evaluates the extent to which smaller datasets (of very complex pattern, e.g harbour megacities) can be used to improve globally applicable CNNs within a specific region. Multispectral imagery from the European Space Agency, Sentinel-2 satellite (10 m spatial resolution) was sourced. Test sites were
selected in Florida, New York, and Shanghai to represent a globally diverse range of waterbody typologies. Region-specific spectral water segmentation algorithms were developed on each test site, to represent benchmarks of spectral index performance. DeepLabV3-ResNet101 was trained on 33,311 semantically labelled true-colour samples. The resulting model was retrained on three smaller subsets of the data, specific to New York, Shanghai and Florida. CNN predictions reached a maximum mean intersection over union result of 0.986 and F1-Score of 0.983. At the Shanghai test site, the CNN’s predictions outperformed the spectral benchmark, primarily due to the CNN’s ability to process contextual features at multiple scales. In all test cases, retraining the networks to localized subsets of the dataset improved the localized region’s segmentation predictions. The CNN’s presented are suitable for cloud-based deployment and could contribute to the wider use of satellite imagery for water management.},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
Near-real time water segmentation with medium resolution satellite imagery plays a critical role in water management. Automated water segmentation of satellite imagery has traditionally been achieved using spectral indices. Spectral water segmentation is limited by environmental factors and requires human expertise to be applied effectively. In recent years, the use of convolutional neural networks (CNN’s) for water segmentation has been successful when used on high-resolution satellite imagery, but to a lesser extent for medium resolution imagery. Existing studies have been limited to geographically localized datasets and reported metrics have been benchmarked against a limited range of spectral indices. This study seeks to determine if a single CNN based on Red, Green, Blue (RGB) image classification can effectively segment water on a global scale and outperform traditional spectral methods. Additionally, this study evaluates the extent to which smaller datasets (of very complex pattern, e.g harbour megacities) can be used to improve globally applicable CNNs within a specific region. Multispectral imagery from the European Space Agency, Sentinel-2 satellite (10 m spatial resolution) was sourced. Test sites were
selected in Florida, New York, and Shanghai to represent a globally diverse range of waterbody typologies. Region-specific spectral water segmentation algorithms were developed on each test site, to represent benchmarks of spectral index performance. DeepLabV3-ResNet101 was trained on 33,311 semantically labelled true-colour samples. The resulting model was retrained on three smaller subsets of the data, specific to New York, Shanghai and Florida. CNN predictions reached a maximum mean intersection over union result of 0.986 and F1-Score of 0.983. At the Shanghai test site, the CNN’s predictions outperformed the spectral benchmark, primarily due to the CNN’s ability to process contextual features at multiple scales. In all test cases, retraining the networks to localized subsets of the dataset improved the localized region’s segmentation predictions. The CNN’s presented are suitable for cloud-based deployment and could contribute to the wider use of satellite imagery for water management.
selected in Florida, New York, and Shanghai to represent a globally diverse range of waterbody typologies. Region-specific spectral water segmentation algorithms were developed on each test site, to represent benchmarks of spectral index performance. DeepLabV3-ResNet101 was trained on 33,311 semantically labelled true-colour samples. The resulting model was retrained on three smaller subsets of the data, specific to New York, Shanghai and Florida. CNN predictions reached a maximum mean intersection over union result of 0.986 and F1-Score of 0.983. At the Shanghai test site, the CNN’s predictions outperformed the spectral benchmark, primarily due to the CNN’s ability to process contextual features at multiple scales. In all test cases, retraining the networks to localized subsets of the dataset improved the localized region’s segmentation predictions. The CNN’s presented are suitable for cloud-based deployment and could contribute to the wider use of satellite imagery for water management.
Subsequence Based Deep Active Learning for Named Entity Recognition Inproceedings
Radmard, Puria; Fathullah, Yassir; Lipani, Aldo
In: Proceedings of the Association for Computational Linguistics, 2021.
@inproceedings{Radmard2021,
title = {Subsequence Based Deep Active Learning for Named Entity Recognition},
author = {Puria Radmard and Yassir Fathullah and Aldo Lipani},
url = {https://www.researchgate.net/publication/351885762_Subsequence_Based_Deep_Active_Learning_for_Named_Entity_Recognition},
year = {2021},
date = {2021-01-01},
booktitle = {Proceedings of the Association for Computational Linguistics},
series = {ACL '21},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Inference of virtual network functions' state via analysis of the CPU behavior Inproceedings
Shelbourne, Charles; Linguaglossa, Leonardo; Zhang, Tianzhu; Lipani, Aldo
In: the 33nd International Teletraffic Congress (ITC 33), 2021.
@inproceedings{Ling2108:Inference,
title = {Inference of virtual network functions' state via analysis of the CPU behavior},
author = {Charles Shelbourne and Leonardo Linguaglossa and Tianzhu Zhang and Aldo Lipani},
url = {https://www.researchgate.net/publication/352826750_Inference_of_virtual_network_functions'_state_via_analysis_of_the_CPU_behavior},
year = {2021},
date = {2021-01-01},
booktitle = {the 33nd International Teletraffic Congress (ITC 33)},
abstract = {The on-going process of softwarization of IT networks promises to reduce the operational and management costs of network infrastructures by replacing hardware middleboxes with equivalent pieces of code executed on general-purpose servers. Alongside the benefits from the operator’s perspective, new strategies to provide the network’s resources to users are arising. Following the principle of “everything as a service”, multiple tenants can access the required resources \textendash typically CPUs, NICs, or RAM \textendash according to a Service-Level Agreement. However, tenants’ applications may require a complex and expensive measurement infrastructure to continuously monitor the network function’s state. Although the application’s specific behavior is unknown (and often opaque to the infrastructure owner), the software nature of (virtual) network functions (VNFs) may be the key to infer the behavior of the high-level functions by accessing low-level information, which is still under the control of the operating system and therefore of the infrastructure owner. As such, in the scenario of software VNFs executed on COTS servers, the underlying CPU’s behavior can be used as the sole predictor for the high-level VNF state without explicit in-network measurements: in this paper, we develop a novel methodology to infer high-level characteristics such as throughput or packet loss using CPU data instead of network measurements. Our methodology consists of (i) experimentally analyzing the behavior of a CPU that executes a VNF under different loads, (ii) extracting a correlation between the CPU footprint and the high-level application state, and (iii) use this knowledge to detect the previously mentioned network metrics. Our code and datasets are publicly available.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
The on-going process of softwarization of IT networks promises to reduce the operational and management costs of network infrastructures by replacing hardware middleboxes with equivalent pieces of code executed on general-purpose servers. Alongside the benefits from the operator’s perspective, new strategies to provide the network’s resources to users are arising. Following the principle of “everything as a service”, multiple tenants can access the required resources – typically CPUs, NICs, or RAM – according to a Service-Level Agreement. However, tenants’ applications may require a complex and expensive measurement infrastructure to continuously monitor the network function’s state. Although the application’s specific behavior is unknown (and often opaque to the infrastructure owner), the software nature of (virtual) network functions (VNFs) may be the key to infer the behavior of the high-level functions by accessing low-level information, which is still under the control of the operating system and therefore of the infrastructure owner. As such, in the scenario of software VNFs executed on COTS servers, the underlying CPU’s behavior can be used as the sole predictor for the high-level VNF state without explicit in-network measurements: in this paper, we develop a novel methodology to infer high-level characteristics such as throughput or packet loss using CPU data instead of network measurements. Our methodology consists of (i) experimentally analyzing the behavior of a CPU that executes a VNF under different loads, (ii) extracting a correlation between the CPU footprint and the high-level application state, and (iii) use this knowledge to detect the previously mentioned network metrics. Our code and datasets are publicly available.
A Baseline for Shapley Values in MLPs: from Missingness to Neutrality Inproceedings
Izzo, Cosimo; Lipani, Aldo; Okhrati, Ramin; Medda, Francesca
In: ESANN, 2021, ISSN: 2331-8422.
@inproceedings{Izzo2021,
title = {A Baseline for Shapley Values in MLPs: from Missingness to Neutrality},
author = {Cosimo Izzo and Aldo Lipani and Ramin Okhrati and Francesca Medda},
url = {http://www.i6doc.com/en/. http://arxiv.org/abs/2006.04896},
issn = {2331-8422},
year = {2021},
date = {2021-01-01},
booktitle = {ESANN},
abstract = {Being able to explain a prediction as well as having a model that performs well are paramount in many machine learning applications. Deep neural networks have gained momentum recently on the basis of their accuracy, however these are often criticised to be black-boxes. Many authors have focused on proposing methods to explain their predictions. Among these explainability methods, feature attribution methods have been favoured for their strong theoretical foundation: the Shapley value. A limitation of Shapley value is the need to define a baseline (aka reference point) representing the missingness of a feature. In this paper, we present a method to choose a baseline based on a neutrality value: a parameter defined by decision makers at which their choices are determined by the returned value of the model being either below or above it. Based on this concept, we theoretically justify these neutral baselines and find a way to identify them for MLPs. Then, we experimentally demonstrate that for a binary classification task, using a synthetic dataset and a dataset coming from the financial domain, the proposed baselines outperform, in terms of local explanability power, standard ways of choosing them.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Being able to explain a prediction as well as having a model that performs well are paramount in many machine learning applications. Deep neural networks have gained momentum recently on the basis of their accuracy, however these are often criticised to be black-boxes. Many authors have focused on proposing methods to explain their predictions. Among these explainability methods, feature attribution methods have been favoured for their strong theoretical foundation: the Shapley value. A limitation of Shapley value is the need to define a baseline (aka reference point) representing the missingness of a feature. In this paper, we present a method to choose a baseline based on a neutrality value: a parameter defined by decision makers at which their choices are determined by the returned value of the model being either below or above it. Based on this concept, we theoretically justify these neutral baselines and find a way to identify them for MLPs. Then, we experimentally demonstrate that for a binary classification task, using a synthetic dataset and a dataset coming from the financial domain, the proposed baselines outperform, in terms of local explanability power, standard ways of choosing them.
Towards More Accountable Search Engines: Online Evaluation of Representation Bias Journal Article
Lipani, Aldo; Piroi, Florina; Yilmaz, Emine
In: arXiv preprint arXiv:2110.08835, 2021.
@article{lipani2021towards,
title = {Towards More Accountable Search Engines: Online Evaluation of Representation Bias},
author = {Aldo Lipani and Florina Piroi and Emine Yilmaz},
url = {https://arxiv.org/abs/2110.08835},
year = {2021},
date = {2021-01-01},
urldate = {2021-01-01},
journal = {arXiv preprint arXiv:2110.08835},
abstract = {Information availability affects people's behavior and perception of the world. Notably, people rely on search engines to satisfy their need for information. Search engines deliver results relevant to user requests usually without being or making themselves accountable for the information they deliver, which may harm people's lives and, in turn, society. This potential risk urges the development of evaluation mechanisms of bias in order to empower the user in judging the results of search engines. In this paper, we give a possible solution to measuring representation bias with respect to societal features for search engines and apply it to evaluating the gender representation bias for Google's Knowledge Graph Carousel for listing occupations.},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
Information availability affects people's behavior and perception of the world. Notably, people rely on search engines to satisfy their need for information. Search engines deliver results relevant to user requests usually without being or making themselves accountable for the information they deliver, which may harm people's lives and, in turn, society. This potential risk urges the development of evaluation mechanisms of bias in order to empower the user in judging the results of search engines. In this paper, we give a possible solution to measuring representation bias with respect to societal features for search engines and apply it to evaluating the gender representation bias for Google's Knowledge Graph Carousel for listing occupations.
2020
A Multilinear Sampling Algorithm to Estimate Shapley Values Inproceedings
Okhrati, Ramin; Lipani, Aldo
In: Proc. of ICPR, 2020.
@inproceedings{okhrati2020multilinear,
title = {A Multilinear Sampling Algorithm to Estimate Shapley Values},
author = {Ramin Okhrati and Aldo Lipani},
url = {https://www.researchgate.net/publication/344825957_A_Multilinear_Sampling_Algorithm_to_Estimate_Shapley_Values},
year = {2020},
date = {2020-10-25},
booktitle = {Proc.~of ICPR},
series = {ICPR},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Predicting Engagement in Video Lectures Inproceedings
Bulathwela, Sahan; Perez-Ortiz, Maria; Lipani, Aldo; Yilmaz, Emine; Shawe-Taylor, John
In: Proc. of EDM, 2020.
@inproceedings{bulathwela2020predictingb,
title = {Predicting Engagement in Video Lectures},
author = {Sahan Bulathwela and Maria Perez-Ortiz and Aldo Lipani and Emine Yilmaz and John Shawe-Taylor},
url = {https://www.researchgate.net/publication/344832087_Predicting_Engagement_in_Video_Lectures},
year = {2020},
date = {2020-01-01},
booktitle = {Proc.~of EDM},
series = {EDM '20},
abstract = {The explosion of Open Educational Resources (OERs) in the recent years creates the demand for scalable, automatic approaches to process and evaluate OERs, with the end goal of identifying and recommending the most suitable educational materials for learners. We focus on building models to find the characteristics and features involved in context-agnostic engagement (i.e. population-based), a seldom researched topic compared to other contextualised and per-sonalised approaches that focus more on individual learner engagement. Learner engagement, is arguably a more reliable measure than popularity/number of views, is more abundant than user ratings and has also been shown to be a crucial component in achieving learning outcomes. In this work, we explore the idea of building a predictive model for population-based engagement in education. We introduce a novel, large dataset of video lectures for predicting context-agnostic engagement and propose both cross-modal and modality specific feature sets to achieve this task. We further test different strategies for quantifying learner engagement signals. We demonstrate the use of our approach in the case of data scarcity. Additionally, we perform a sensitivity analysis of the best performing model, which shows promising performance and can be easily integrated into an educational recommender system for OERs.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
The explosion of Open Educational Resources (OERs) in the recent years creates the demand for scalable, automatic approaches to process and evaluate OERs, with the end goal of identifying and recommending the most suitable educational materials for learners. We focus on building models to find the characteristics and features involved in context-agnostic engagement (i.e. population-based), a seldom researched topic compared to other contextualised and per-sonalised approaches that focus more on individual learner engagement. Learner engagement, is arguably a more reliable measure than popularity/number of views, is more abundant than user ratings and has also been shown to be a crucial component in achieving learning outcomes. In this work, we explore the idea of building a predictive model for population-based engagement in education. We introduce a novel, large dataset of video lectures for predicting context-agnostic engagement and propose both cross-modal and modality specific feature sets to achieve this task. We further test different strategies for quantifying learner engagement signals. We demonstrate the use of our approach in the case of data scarcity. Additionally, we perform a sensitivity analysis of the best performing model, which shows promising performance and can be easily integrated into an educational recommender system for OERs.
Self-Attentive Hawkes Processes Inproceedings
Zhang, Qiang; Lipani, Aldo; Kirnap, Omer; Yilmaz, Emine
In: Proc. of ICML, 2020.
@inproceedings{zhang2020selfattentiveb,
title = {Self-Attentive Hawkes Processes},
author = {Qiang Zhang and Aldo Lipani and Omer Kirnap and Emine Yilmaz},
url = {https://www.researchgate.net/publication/344832377_Self-Attentive_Hawkes_Process},
year = {2020},
date = {2020-01-01},
booktitle = {Proc.~of ICML},
series = {ICML '20},
abstract = {Capturing the occurrence dynamics is crucial to predicting which type of events will happen next and when. A common method to do this is through Hawkes processes. To enhance their capacity, recurrent neural networks (RNNs) have been incorporated due to RNNs’ successes in processing sequential data such as languages. Recent evidence suggests that self-attention is more competent than RNNs in dealing with languages. However, we are unaware of the effectiveness of self-attention in the context of Hawkes processes. This study aims to fill the gap by designing a self-attentive Hawkes process (SAHP). SAHP employs self-attention to summarise the influence of history events and compute the probability of the next event. One deficit of the conventional self-attention when applied to event sequences is that its positional encoding only considers the order of a sequence ignoring the time intervals between events. To overcome this deficit, we modify its encoding by translating time intervals into phase shifts of sinusoidal functions. Experiments on goodness-of-fit and prediction tasks show the improved capability of SAHP. Furthermore, SAHP is more interpretable than RNN-based counterparts because the learnt attention weights reveal contributions of one event type to the happening of another type. To the best of our knowledge, this is the first work that studies the effectiveness of self-attention in Hawkes processes.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Capturing the occurrence dynamics is crucial to predicting which type of events will happen next and when. A common method to do this is through Hawkes processes. To enhance their capacity, recurrent neural networks (RNNs) have been incorporated due to RNNs’ successes in processing sequential data such as languages. Recent evidence suggests that self-attention is more competent than RNNs in dealing with languages. However, we are unaware of the effectiveness of self-attention in the context of Hawkes processes. This study aims to fill the gap by designing a self-attentive Hawkes process (SAHP). SAHP employs self-attention to summarise the influence of history events and compute the probability of the next event. One deficit of the conventional self-attention when applied to event sequences is that its positional encoding only considers the order of a sequence ignoring the time intervals between events. To overcome this deficit, we modify its encoding by translating time intervals into phase shifts of sinusoidal functions. Experiments on goodness-of-fit and prediction tasks show the improved capability of SAHP. Furthermore, SAHP is more interpretable than RNN-based counterparts because the learnt attention weights reveal contributions of one event type to the happening of another type. To the best of our knowledge, this is the first work that studies the effectiveness of self-attention in Hawkes processes.
Easing Legal News Monitoring with Learning to Rank and BERT Inproceedings
Sanchez, Luis; He, Jiyin; Manotumruksa, Jarana; Albakour, Dyaa; Martinez, Miguel; Lipani, Aldo
In: Proc. of ECIR, pp. 336–343, Springer International Publishing, Cham, 2020.
@inproceedings{10.1007/978-3-030-45442-5_42,
title = {Easing Legal News Monitoring with Learning to Rank and BERT},
author = {Luis Sanchez and Jiyin He and Jarana Manotumruksa and Dyaa Albakour and Miguel Martinez and Aldo Lipani},
url = {https://www.researchgate.net/publication/338825714_Easing_Legal_News_Monitoring_with_Learning_to_Rank_and_BERT},
year = {2020},
date = {2020-01-01},
booktitle = {Proc.~of ECIR},
pages = {336--343},
publisher = {Springer International Publishing},
address = {Cham},
series = {ECIR '20},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Learning to Re-Rank with Contextualized Stopwords Inproceedings
Hofstätter, Sebastian; Lipani, Aldo; Zlabinger, Markus; Hanbury, Allan
In: Proc. of CIKM, 2020.
@inproceedings{hofstaetter2020learning,
title = {Learning to Re-Rank with Contextualized Stopwords},
author = {Sebastian Hofst\"{a}tter and Aldo Lipani and Markus Zlabinger and Allan Hanbury},
url = {https://www.researchgate.net/publication/344832244_Learning_to_Re-Rank_with_Contextualized_Stopwords},
year = {2020},
date = {2020-01-01},
booktitle = {Proc.~of CIKM},
series = {CIKM '20},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
2019
FACTS-IR: Fairness, Accountability, Confidentiality, Transparency, and Safety in Information Retrieval Journal Article
Roegiest, Adam; Lipani, Aldo; Beutel, Alex; Olteanu, Alexandra; Lucic, Ana; Stoica, Ana-Andreea; Das, Anubrata; Biega, Asia; Voorn, Bart; Hauff, Claudia; Spina, Damiano; Lewis, David; Oard, Douglas W; Yilmaz, Emine; Hasibi, Faegheh; Kazai, Gabriella; McDonald, Graham; Haned, Hinda; Ounis, Iadh; van der Linden, Ilse; Garcia-Gathright, Jean; Baan, Joris; Lau, Kamuela N; Balog, Krisztian; de Rijke, Maarten; Sayed, Mahmoud; Panteli, Maria; Sanderson, Mark; Lease, Matthew; Ekstrand, Michael D; Lahoti, Preethi; Kamishima, Toshihiro
In: SIGIR Forum, vol. 53, no. 2, pp. 20–43, 2019.
@article{roegiest-2019-facts-ir,
title = {FACTS-IR: Fairness, Accountability, Confidentiality, Transparency, and Safety in Information Retrieval},
author = {Adam Roegiest and Aldo Lipani and Alex Beutel and Alexandra Olteanu and Ana Lucic and Ana-Andreea Stoica and Anubrata Das and Asia Biega and Bart Voorn and Claudia Hauff and Damiano Spina and David Lewis and Douglas W Oard and Emine Yilmaz and Faegheh Hasibi and Gabriella Kazai and Graham McDonald and Hinda Haned and Iadh Ounis and Ilse van der Linden and Jean Garcia-Gathright and Joris Baan and Kamuela N Lau and Krisztian Balog and Maarten de Rijke and Mahmoud Sayed and Maria Panteli and Mark Sanderson and Matthew Lease and Michael D Ekstrand and Preethi Lahoti and Toshihiro Kamishima},
url = {https://www.researchgate.net/publication/337933046_FACTS-IR_Fairness_Accountability_Confidentiality_Transparency_and_Safety_in_Information_Retrieval},
year = {2019},
date = {2019-12-01},
journal = {SIGIR Forum},
volume = {53},
number = {2},
pages = {20--43},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
Fixed-Cost Pooling Strategies Journal Article
Lipani, Aldo; Losada, David E.; Zuccon, Guido; Lupu, Mihai
In: IEEE Transactions on Knowledge and Data Engineering (TKDE), 2019.
@article{Lipani2019TKDE,
title = {Fixed-Cost Pooling Strategies},
author = {Aldo Lipani and David E. Losada and Guido Zuccon and Mihai Lupu},
url = {https://www.researchgate.net/publication/336369355_Fixed-Cost_Pooling_Strategies},
year = {2019},
date = {2019-10-06},
journal = {IEEE Transactions on Knowledge and Data Engineering (TKDE)},
abstract = {The empirical nature of Information Retrieval (IR) mandates strong experimental practices. A keystone of such experimental practices is the Cranfield evaluation paradigm. Within this paradigm, the collection of relevance judgments has been the subject of intense scientific investigation. This is because, on one hand, consistent, precise, and numerous judgements are keys to reducing evaluation uncertainty and test collection bias; on the other hand, however, relevance judgements are costly to collect. The selection of which documents to judge for relevance, known as pooling method, has therefore a great impact on IR evaluation. In this paper we focus on the bias introduced by the pooling method, known as pool bias, which affects the reusability of test collections, in particular when building test collections with a limited budget. In this paper we formalize and evaluate a set of 22 pooling strategies based on: traditional strategies, voting systems, retrieval fusion methods, evaluation measures, and multi-armed bandit models. To do this we run a large-scale evaluation by considering a set of 9 standard TREC test collections, in which we show that the choice of the pooling strategy has significant effects on the cost needed to obtain an unbiased test collection. We also identify the least biased pooling strategy in terms of pool bias according to three IR evaluation measures: AP, NDCG, and P@10.},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
The empirical nature of Information Retrieval (IR) mandates strong experimental practices. A keystone of such experimental practices is the Cranfield evaluation paradigm. Within this paradigm, the collection of relevance judgments has been the subject of intense scientific investigation. This is because, on one hand, consistent, precise, and numerous judgements are keys to reducing evaluation uncertainty and test collection bias; on the other hand, however, relevance judgements are costly to collect. The selection of which documents to judge for relevance, known as pooling method, has therefore a great impact on IR evaluation. In this paper we focus on the bias introduced by the pooling method, known as pool bias, which affects the reusability of test collections, in particular when building test collections with a limited budget. In this paper we formalize and evaluate a set of 22 pooling strategies based on: traditional strategies, voting systems, retrieval fusion methods, evaluation measures, and multi-armed bandit models. To do this we run a large-scale evaluation by considering a set of 9 standard TREC test collections, in which we show that the choice of the pooling strategy has significant effects on the cost needed to obtain an unbiased test collection. We also identify the least biased pooling strategy in terms of pool bias according to three IR evaluation measures: AP, NDCG, and P@10.
From a User Model for Query Sessions to Session Rank Biased Precision (sRBP) Inproceedings
Lipani, Aldo; Carterette, Ben; Yilmaz, Emine
In: Proc. of ICTIR, 2019.
@inproceedings{Lipani2019,
title = {From a User Model for Query Sessions to Session Rank Biased Precision (sRBP)},
author = {Aldo Lipani and Ben Carterette and Emine Yilmaz},
url = {https://www.researchgate.net/publication/334725760_From_a_User_Model_for_Query_Sessions_to_Session_Rank_Biased_Precision_sRBP},
doi = {10.1145/3341981.3344216},
year = {2019},
date = {2019-10-02},
booktitle = {Proc.~of ICTIR},
journal = {Proc.~of ICTIR},
abstract = {To satisfy their information needs, users usually carry out searches on retrieval systems by continuously trading off between the examination of search results retrieved by under-specified queries and the refinement of these queries through reformulation. In Information Retrieval (IR), a series of query reformulations is known as a query-session. Research in IR evaluation has traditionally been focused on the development of measures for the ad hoc task, for which a retrieval system aims to retrieve the best documents for a single query. Thus, most IR evaluation measures, with a few exceptions , are not suitable to evaluate retrieval scenarios that call for multiple refinements over a query-session. In this paper, by formally modeling a user's expected behaviour over query-sessions, we derive a session-based evaluation measure, which results in a generalization of the evaluation measure Rank Biased Precision (RBP). We demonstrate the quality of this new session-based evaluation measure, named Session RBP (sRBP), by evaluating its user model against the observed user behaviour over the query-sessions of the 2014 TREC Session track.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
To satisfy their information needs, users usually carry out searches on retrieval systems by continuously trading off between the examination of search results retrieved by under-specified queries and the refinement of these queries through reformulation. In Information Retrieval (IR), a series of query reformulations is known as a query-session. Research in IR evaluation has traditionally been focused on the development of measures for the ad hoc task, for which a retrieval system aims to retrieve the best documents for a single query. Thus, most IR evaluation measures, with a few exceptions , are not suitable to evaluate retrieval scenarios that call for multiple refinements over a query-session. In this paper, by formally modeling a user's expected behaviour over query-sessions, we derive a session-based evaluation measure, which results in a generalization of the evaluation measure Rank Biased Precision (RBP). We demonstrate the quality of this new session-based evaluation measure, named Session RBP (sRBP), by evaluating its user model against the observed user behaviour over the query-sessions of the 2014 TREC Session track.
Reply-aided Detection of Misinformation via Bayesian Deep Learning Inproceedings
Zhang, Qiang; Lipani, Aldo; Liang, Shangsong; Yilmaz, Emine
In: Proc. of WWW, 2019.
@inproceedings{Zhang2019b,
title = {Reply-aided Detection of Misinformation via Bayesian Deep Learning},
author = {Qiang Zhang and Aldo Lipani and Shangsong Liang and Emine Yilmaz},
doi = {10.1145/3308558.3313718},
year = {2019},
date = {2019-05-13},
booktitle = {Proc.~of WWW},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
From Stances’ Imbalance to Their Hierarchical Representation and Detection Inproceedings
Zhang, Qiang; Liang, Shangsong; Lipani, Aldo; Ren, Zhaochun; Yilmaz, Emine
In: Proc. of WWW, 2019.
@inproceedings{Zhang2019b,
title = {From Stances’ Imbalance to Their Hierarchical Representation and Detection},
author = {Qiang Zhang and Shangsong Liang and Aldo Lipani and Zhaochun Ren and Emine Yilmaz},
doi = {10.1145/3308558.3313724},
year = {2019},
date = {2019-05-13},
booktitle = {Proc.~of WWW},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
On Biases in Information Retrieval Models and Evaluation Journal Article
Lipani, Aldo
In: SIGIR Forum, vol. 52, no. 2, 2019.
@article{Lipani:2019:BIR:3308774.3308804,
title = {On Biases in Information Retrieval Models and Evaluation},
author = {Aldo Lipani},
doi = {10.1145/3308774.3308804},
year = {2019},
date = {2019-01-01},
journal = {SIGIR Forum},
volume = {52},
number = {2},
publisher = {ACM},
address = {New York, NY, USA},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
On the Learnability of Software Router Performance via CPU Measurements Inproceedings
Shelbourne, Charles; Linguaglossa, Leonardo; Lipani, Aldo; Zhang, Tianzhu; Geyer, Fabien
In: Proc. of CoNEXT, pp. 23–25, Association for Computing Machinery, Orlando, FL, USA, 2019, ISBN: 9781450370066.
@inproceedings{10.1145/3360468.3366776,
title = {On the Learnability of Software Router Performance via CPU Measurements},
author = {Charles Shelbourne and Leonardo Linguaglossa and Aldo Lipani and Tianzhu Zhang and Fabien Geyer},
url = {https://www.researchgate.net/publication/337580746_On_the_Learnability_of_Software_Router_Performance_via_CPU_Measurements
https://doi.org/10.1145/3360468.3366776
},
doi = {10.1145/3360468.3366776},
isbn = {9781450370066},
year = {2019},
date = {2019-01-01},
booktitle = {Proc.~of CoNEXT},
pages = {23\textendash25},
publisher = {Association for Computing Machinery},
address = {Orlando, FL, USA},
series = {CoNEXT ’19},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
2018
On Biases in Information Retrieval Models and Evaluation PhD Thesis
Lipani, Aldo
TU Wien, 2018.
@phdthesis{PhDLipani2018,
title = {On Biases in Information Retrieval Models and Evaluation},
author = {Aldo Lipani},
url = {http://aldolipani.com/wp-content/uploads/2018/09/phd_thesis.pdf},
doi = {10.13140/RG.2.2.28623.74400},
year = {2018},
date = {2018-09-21},
school = {TU Wien},
abstract = {The advent of the modern information technology has benefited society as the digitisation of content increased over the last half-century. While the processing capability of our species has remained unchanged, the information available to us has been notably increasing. In this overload of information, Information Retrieval (IR) has been playing a prominent role by developing systems capable of separating relevant information from the rest. This separation, however, is a difficult task rooted in the complexity of understanding of what is and what is not relevant. To manage this complexity, IR has developed a strong empirical nature, which has led to the development of grounded retrieval models, resulting in the development of retrieval systems empirically designed to be biased towards relevant information. However, other biases have been observed, which counteract retrieval performance. In this thesis, the reduction of retrieval systems to filters of information, or sampling processes, has allowed us to systematically investigate these biases.
We study biases manifesting in two aspects of IR research: retrieval models and retrieval evaluation. We start by identifying retrieval biases in probabilistic IR models and then develop new document priors to improve retrieval performance. Next, we discuss the accessibility bias of retrieval models, and for Boolean retrieval models we develop a mathematical framework of retrievability. For retrieval evaluation biases, we study how test collections are built using the pooling method and how this method introduces bias. Then, to improve the reliability of the evaluation, we first develop new pooling strategies to mitigate this bias at test collection build time and then, for two IR evaluation measures, Precision and Recall at cut-off (P@n and R@n), we develop new pool bias estimators to mitigate it at evaluation time.
Through a large scale experimentation involving up to 15 test collections, four IR evaluation measures and three bias measures, we demonstrate that including document priors based on verboseness improves the performance of probabilistic retrieval models; that the accessibility bias of Boolean retrieval models quickly worsens for conjunctive queries with the increase of the query length (while slightly improving for disjunctive queries); that the test collection bias can be lowered at test collection build time by pooling strategies inspired by a well-known problem in reinforcement learning, the multi-armed bandit problem; and that this bias can also be improved at evaluation time by analysing the runs participating in the pool. For this last point in particular, we show that for P@n, bias reduction is done by quantifying the potential of the new system against the pooled runs, and for R@n, this is done instead by simulating the absence of a pooled run from the set of pooled runs.
This thesis contributes to the IR field by giving a better understanding of relevance through the lens of biases in retrieval models and retrieval evaluation. The identification of these biases, and their exploitation or mitigation, leads to the development of better performing IR models and the improvement of the current IR evaluation practice.},
keywords = {},
pubstate = {published},
tppubtype = {phdthesis}
}
The advent of the modern information technology has benefited society as the digitisation of content increased over the last half-century. While the processing capability of our species has remained unchanged, the information available to us has been notably increasing. In this overload of information, Information Retrieval (IR) has been playing a prominent role by developing systems capable of separating relevant information from the rest. This separation, however, is a difficult task rooted in the complexity of understanding of what is and what is not relevant. To manage this complexity, IR has developed a strong empirical nature, which has led to the development of grounded retrieval models, resulting in the development of retrieval systems empirically designed to be biased towards relevant information. However, other biases have been observed, which counteract retrieval performance. In this thesis, the reduction of retrieval systems to filters of information, or sampling processes, has allowed us to systematically investigate these biases.
We study biases manifesting in two aspects of IR research: retrieval models and retrieval evaluation. We start by identifying retrieval biases in probabilistic IR models and then develop new document priors to improve retrieval performance. Next, we discuss the accessibility bias of retrieval models, and for Boolean retrieval models we develop a mathematical framework of retrievability. For retrieval evaluation biases, we study how test collections are built using the pooling method and how this method introduces bias. Then, to improve the reliability of the evaluation, we first develop new pooling strategies to mitigate this bias at test collection build time and then, for two IR evaluation measures, Precision and Recall at cut-off (P@n and R@n), we develop new pool bias estimators to mitigate it at evaluation time.
Through a large scale experimentation involving up to 15 test collections, four IR evaluation measures and three bias measures, we demonstrate that including document priors based on verboseness improves the performance of probabilistic retrieval models; that the accessibility bias of Boolean retrieval models quickly worsens for conjunctive queries with the increase of the query length (while slightly improving for disjunctive queries); that the test collection bias can be lowered at test collection build time by pooling strategies inspired by a well-known problem in reinforcement learning, the multi-armed bandit problem; and that this bias can also be improved at evaluation time by analysing the runs participating in the pool. For this last point in particular, we show that for P@n, bias reduction is done by quantifying the potential of the new system against the pooled runs, and for R@n, this is done instead by simulating the absence of a pooled run from the set of pooled runs.
This thesis contributes to the IR field by giving a better understanding of relevance through the lens of biases in retrieval models and retrieval evaluation. The identification of these biases, and their exploitation or mitigation, leads to the development of better performing IR models and the improvement of the current IR evaluation practice.
We study biases manifesting in two aspects of IR research: retrieval models and retrieval evaluation. We start by identifying retrieval biases in probabilistic IR models and then develop new document priors to improve retrieval performance. Next, we discuss the accessibility bias of retrieval models, and for Boolean retrieval models we develop a mathematical framework of retrievability. For retrieval evaluation biases, we study how test collections are built using the pooling method and how this method introduces bias. Then, to improve the reliability of the evaluation, we first develop new pooling strategies to mitigate this bias at test collection build time and then, for two IR evaluation measures, Precision and Recall at cut-off (P@n and R@n), we develop new pool bias estimators to mitigate it at evaluation time.
Through a large scale experimentation involving up to 15 test collections, four IR evaluation measures and three bias measures, we demonstrate that including document priors based on verboseness improves the performance of probabilistic retrieval models; that the accessibility bias of Boolean retrieval models quickly worsens for conjunctive queries with the increase of the query length (while slightly improving for disjunctive queries); that the test collection bias can be lowered at test collection build time by pooling strategies inspired by a well-known problem in reinforcement learning, the multi-armed bandit problem; and that this bias can also be improved at evaluation time by analysing the runs participating in the pool. For this last point in particular, we show that for P@n, bias reduction is done by quantifying the potential of the new system against the pooled runs, and for R@n, this is done instead by simulating the absence of a pooled run from the set of pooled runs.
This thesis contributes to the IR field by giving a better understanding of relevance through the lens of biases in retrieval models and retrieval evaluation. The identification of these biases, and their exploitation or mitigation, leads to the development of better performing IR models and the improvement of the current IR evaluation practice.
A Systematic Approach to Normalization in Probabilistic Models Journal Article
Lipani, Aldo; Roelleke, Thomas; Lupu, Mihai; Hanbury, Allan
In: Information Retrieval Journal, 2018.
@article{Lipani2018,
title = {A Systematic Approach to Normalization in Probabilistic Models},
author = {Aldo Lipani and Thomas Roelleke and Mihai Lupu and Allan Hanbury},
doi = {10.1007/s10791-018-9334-1},
year = {2018},
date = {2018-06-30},
journal = {Information Retrieval Journal},
abstract = {Every information retrieval (IR) model embeds in its scoring function a form of term frequency (TF) quantification. The contribution of the term frequency is determined by the properties of the function of the chosen TF quantification, and by its TF normalization. The first defines how independent the occurrences of multiple terms are, while the second acts on mitigating the a priori probability of having a high term frequency in a document (estimation usually based on the document length). New test collections, coming from different domains (e.g. medical, legal), give evidence that not only document length, but in addition, verboseness of documents should be explicitly considered. Therefore we propose and investigate a systematic combination of document verboseness and length. To theoretically justify the combination, we show the duality between document verboseness and length. In addition, we investigate the duality between verboseness and other components of IR models. We test these new TF normalizations on four suitable test collections. We do this on a well defined spectrum of TF quantifications. Finally, based on the theoretical and experimental observations, we show how the two components of this new normalization, document verboseness and length, interact with each other. Our experiments demonstrate that the new models never underperform existing models, while sometimes introducing statistically significantly better results, at no additional computational cost.},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
Every information retrieval (IR) model embeds in its scoring function a form of term frequency (TF) quantification. The contribution of the term frequency is determined by the properties of the function of the chosen TF quantification, and by its TF normalization. The first defines how independent the occurrences of multiple terms are, while the second acts on mitigating the a priori probability of having a high term frequency in a document (estimation usually based on the document length). New test collections, coming from different domains (e.g. medical, legal), give evidence that not only document length, but in addition, verboseness of documents should be explicitly considered. Therefore we propose and investigate a systematic combination of document verboseness and length. To theoretically justify the combination, we show the duality between document verboseness and length. In addition, we investigate the duality between verboseness and other components of IR models. We test these new TF normalizations on four suitable test collections. We do this on a well defined spectrum of TF quantifications. Finally, based on the theoretical and experimental observations, we show how the two components of this new normalization, document verboseness and length, interact with each other. Our experiments demonstrate that the new models never underperform existing models, while sometimes introducing statistically significantly better results, at no additional computational cost.
2017
Fixed-Cost Pooling Strategies Based on IR Evaluation Measures Inproceedings
Lipani, Aldo; Palotti, Joao; Lupu, Mihai; Piroi, Florina; Zuccon, Guido; Hanbury, Allan
In: Proc. of ECIR, 2017.
@inproceedings{Lipani2017,
title = {Fixed-Cost Pooling Strategies Based on IR Evaluation Measures},
author = {Aldo Lipani and Joao Palotti and Mihai Lupu and Florina Piroi and Guido Zuccon and Allan Hanbury},
doi = {10.1007/978-3-319-56608-5_28},
year = {2017},
date = {2017-01-01},
booktitle = {Proc.~of ECIR},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Visual Pool: A Tool to Visualize and Interact with the Pooling Method Inproceedings
Lipani, Aldo; Lupu, Mihai; Hanbury, Allan
In: Proc. of SIGIR, 2017.
@inproceedings{Lipani:2017:VPT:3077136.3084146,
title = {Visual Pool: A Tool to Visualize and Interact with the Pooling Method},
author = {Aldo Lipani and Mihai Lupu and Allan Hanbury},
doi = {10.1145/3077136.3084146},
year = {2017},
date = {2017-01-01},
booktitle = {Proc.~of SIGIR},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Fixed Budget Pooling Strategies Based on Fusion Methods Inproceedings
Lipani, Aldo; Lupu, Mihai; Palotti, Joao; Zuccon, Guido; Hanbury, Allan
In: Proc. of SAC, 2017.
@inproceedings{Lipani:2017:FBP:3019612.3019692,
title = {Fixed Budget Pooling Strategies Based on Fusion Methods},
author = {Aldo Lipani and Mihai Lupu and Joao Palotti and Guido Zuccon and Allan Hanbury},
doi = {10.1145/3019612.3019692},
year = {2017},
date = {2017-01-01},
booktitle = {Proc.~of SAC},
abstract = {The empirical nature of Information Retrieval (IR) mandates strong experimental practices. The Cranfield/TREC evaluation paradigm represents a keystone of such experimental practices. Within this paradigm, the generation of relevance judgments has been the subject of intense scientific investigation. This is because, on one hand, consistent, precise and numerous judgements are key to reduce evaluation uncertainty and test collection bias; on the other hand, however, relevance judgements are costly to collect. The selection of which documents to judge for relevance (known as pooling) has therefore great impact in IR evaluation. In this paper, we contribute a set of 8 novel pooling strategies based on retrieval fusion methods. We show that the choice of the pooling strategy has significant effects on the cost needed to obtain an unbiased test collection; we also identify the best performing pooling strategy according to three evaluation measure.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
The empirical nature of Information Retrieval (IR) mandates strong experimental practices. The Cranfield/TREC evaluation paradigm represents a keystone of such experimental practices. Within this paradigm, the generation of relevance judgments has been the subject of intense scientific investigation. This is because, on one hand, consistent, precise and numerous judgements are key to reduce evaluation uncertainty and test collection bias; on the other hand, however, relevance judgements are costly to collect. The selection of which documents to judge for relevance (known as pooling) has therefore great impact in IR evaluation. In this paper, we contribute a set of 8 novel pooling strategies based on retrieval fusion methods. We show that the choice of the pooling strategy has significant effects on the cost needed to obtain an unbiased test collection; we also identify the best performing pooling strategy according to three evaluation measure.
Schillaci, Calogero; Acutis, Marco; Lombardo, Luigi; Lipani, Aldo; Fantappiè, Maria; Märker, Michael; Saia, Sergio
In: Science of The Total Environment, vol. 601-602, 2017.
@article{SCHILLACI2017821,
title = {Spatio-temporal topsoil organic carbon mapping of a semi-arid Mediterranean region: The role of land use, soil texture, topographic indices and the influence of remote sensing data to modelling},
author = {Calogero Schillaci and Marco Acutis and Luigi Lombardo and Aldo Lipani and Maria Fantappi\`{e} and Michael M\"{a}rker and Sergio Saia},
doi = {10.1016/j.scitotenv.2017.05.239},
year = {2017},
date = {2017-01-01},
journal = {Science of The Total Environment},
volume = {601-602},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
2016
Fairness in Information Retrieval Inproceedings
Lipani, Aldo
In: Proc. of SIGIR, 2016.
@inproceedings{Lipani:2016:FIR:2911451.2911473,
title = {Fairness in Information Retrieval},
author = {Aldo Lipani},
doi = {10.1145/2911451.2911473},
year = {2016},
date = {2016-01-01},
booktitle = {Proc.~of SIGIR},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
The Solitude of Relevant Documents in the Pool Inproceedings
Lipani, Aldo; Lupu, Mihai; Kanoulas, Evangelos; Hanbury, Allan
In: Proc. of CIKM, 2016.
@inproceedings{Lipani:2016:SRD:2983323.2983891,
title = {The Solitude of Relevant Documents in the Pool},
author = {Aldo Lipani and Mihai Lupu and Evangelos Kanoulas and Allan Hanbury},
doi = {10.1145/2983323.2983891},
year = {2016},
date = {2016-01-01},
booktitle = {Proc.~of CIKM},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
The Curious Incidence of Bias Corrections in the Pool Inproceedings
Lipani, Aldo; Lupu, Mihai; Hanbury, Allan
In: Proc. of ECIR, 2016.
@inproceedings{Lipani2016,
title = {The Curious Incidence of Bias Corrections in the Pool},
author = {Aldo Lipani and Mihai Lupu and Allan Hanbury},
doi = {10.1007/978-3-319-30671-1_20},
year = {2016},
date = {2016-01-01},
booktitle = {Proc.~of ECIR},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}