@article{BarbosaFMMKCS25,
  title = {Attributes of a great requirements engineer},
  journal = {Journal of Systems and Software},
  volume = {219},
  pages = {112200},
  year = {2025},
  issn = {0164-1212},
  doi = {https://doi.org/10.1016/j.jss.2024.112200},
  urlAuthor_version = {http://www.inf.puc-rio.br/~kalinowski/publications/BarbosaFMMKCS25.pdf},
  author = {Larissa Barbosa and Sávio Freire and Rita S.P. Maciel and Manoel Mendonça and Marcos Kalinowski and Zadia Codabux and Rodrigo Spínola},
  keywords = {Requirements Engineering, Attributes of a great requirements engineer, Skill, Survey},
  abstract = {Context and motivation:
  Several studies have investigated attributes of great software practitioners. However, the investigation of such attributes is still missing in Requirements Engineering (RE). The current knowledge on attributes of great software practitioners might not be easily translated to the context of RE because its activities are, usually, less technical and more human-centered than other software engineering activities.
  Question/problem:
  This work aims to investigate which are the attributes of great requirements engineers, the relationship between them, and strategies that can be employed to obtain these attributes. We follow a method composed of a survey with 18 practitioners and follow up interviews with 11 of them.
  Principal ideas/results:
  Investigative ability in talking to stakeholders, judicious, and understand the business are the most commonly mentioned attributes amongst the set of 22 attributes identified, which were grouped into four categories. We also found 38 strategies to improve RE skills. Examples are training, talking to all stakeholders, and acquiring domain knowledge.
  Contribution:
  The attributes, their categories, and relationships are organized into a map. The relations between attributes and strategies are represented in a Sankey diagram. Software practitioners can use our findings to improve their understanding about the role and responsibilities of requirements engineers.},
}

Context and motivation: Several studies have investigated attributes of great software practitioners. However, the investigation of such attributes is still missing in Requirements Engineering (RE). The current knowledge on attributes of great software practitioners might not be easily translated to the context of RE because its activities are, usually, less technical and more human-centered than other software engineering activities. Question/problem: This work aims to investigate which are the attributes of great requirements engineers, the relationship between them, and strategies that can be employed to obtain these attributes. We follow a method composed of a survey with 18 practitioners and follow up interviews with 11 of them. Principal ideas/results: Investigative ability in talking to stakeholders, judicious, and understand the business are the most commonly mentioned attributes amongst the set of 22 attributes identified, which were grouped into four categories. We also found 38 strategies to improve RE skills. Examples are training, talking to all stakeholders, and acquiring domain knowledge. Contribution: The attributes, their categories, and relationships are organized into a map. The relations between attributes and strategies are represented in a Sankey diagram. Software practitioners can use our findings to improve their understanding about the role and responsibilities of requirements engineers.

@article{QuarantaACK25,
  title = {A multivocal literature review on the benefits and limitations of industry-leading AutoML tools},
  journal = {Information and Software Technology},
  volume = {178},
  pages = {107608},
  year = {2025},
  issn = {0950-5849},
  doi = {https://doi.org/10.1016/j.infsof.2024.107608},
  urlAuthor_version = {http://www.inf.puc-rio.br/~kalinowski/publications/QuarantaACK25.pdf},
  author = {Luigi Quaranta and Kelly Azevedo and Fabio Calefato and Marcos Kalinowski},
  keywords = {Multivocal literature review, Automl, Autoai, Benefits, Limitations},
  abstract = {Context:
  Rapid advancements in Artificial Intelligence (AI) and Machine Learning (ML) are revolutionizing software engineering in every application domain, driving unprecedented transformations and fostering innovation. However, despite these advances, several organizations are experiencing friction in the adoption of ML-based technologies, mainly due to the current shortage of ML professionals. In this context, Automated Machine Learning (AutoML) techniques have been presented as a promising solution to democratize ML adoption, even in the absence of specialized people.
  Objective:
  Our research aims to provide an overview of the evidence on the benefits and limitations of AutoML tools being adopted in industry.
  Methods:
  We conducted a Multivocal Literature Review, which allowed us to identify 54 sources from the academic literature and 108 sources from the grey literature reporting on AutoML benefits and limitations. We extracted explicitly reported benefits and limitations from the papers and applied the thematic analysis method for synthesis.
  Results:
  In general, we identified 18 reported benefits and 25 limitations. Concerning the benefits, we highlight that AutoML tools can help streamline the core steps of ML workflows, namely data preparation, feature engineering, model construction, and hyperparameter tuning—with concrete benefits on model performance, efficiency, and scalability. In addition, AutoML empowers both novice and experienced data scientists, promoting ML accessibility. However, we highlight several limitations that may represent obstacles to the widespread adoption of AutoML. For instance, AutoML tools may introduce barriers to transparency and interoperability, exhibit limited flexibility for complex scenarios, and offer inconsistent coverage of the ML workflow.
  Conclusion:
  The effectiveness of AutoML in facilitating the adoption of machine learning by users may vary depending on the specific tool and the context in which it is used. Today, AutoML tools are used to increase human expertise rather than replace it and, as such, require skilled users.}
}

Context: Rapid advancements in Artificial Intelligence (AI) and Machine Learning (ML) are revolutionizing software engineering in every application domain, driving unprecedented transformations and fostering innovation. However, despite these advances, several organizations are experiencing friction in the adoption of ML-based technologies, mainly due to the current shortage of ML professionals. In this context, Automated Machine Learning (AutoML) techniques have been presented as a promising solution to democratize ML adoption, even in the absence of specialized people. Objective: Our research aims to provide an overview of the evidence on the benefits and limitations of AutoML tools being adopted in industry. Methods: We conducted a Multivocal Literature Review, which allowed us to identify 54 sources from the academic literature and 108 sources from the grey literature reporting on AutoML benefits and limitations. We extracted explicitly reported benefits and limitations from the papers and applied the thematic analysis method for synthesis. Results: In general, we identified 18 reported benefits and 25 limitations. Concerning the benefits, we highlight that AutoML tools can help streamline the core steps of ML workflows, namely data preparation, feature engineering, model construction, and hyperparameter tuning—with concrete benefits on model performance, efficiency, and scalability. In addition, AutoML empowers both novice and experienced data scientists, promoting ML accessibility. However, we highlight several limitations that may represent obstacles to the widespread adoption of AutoML. For instance, AutoML tools may introduce barriers to transparency and interoperability, exhibit limited flexibility for complex scenarios, and offer inconsistent coverage of the ML workflow. Conclusion: The effectiveness of AutoML in facilitating the adoption of machine learning by users may vary depending on the specific tool and the context in which it is used. Today, AutoML tools are used to increase human expertise rather than replace it and, as such, require skilled users.

@inproceedings{AraujoKB25,
  author = {Araujo, Allysson Allex and Kalinowski, Marcos and Baldassarre, Maria Teresa},
  title = {Embracing Experiential Learning: Hackathons as an Educational Strategy for Shaping Soft Skills in Software Engineering},
  year = {2025},
  isbn = {},
  publisher = {IEEE},
  urlAuthor_version = {http://www.inf.puc-rio.br/~kalinowski/publications/AraujoKB25.pdf},   
  doi = {},
  abstract = {In recent years, Software Engineering (SE) scholars and practitioners have emphasized the importance of integrating soft skills into SE education. However, teaching and learning soft skills are complex, as they cannot be acquired passively through raw knowledge acquisition. On the other hand, hackathons have attracted increasing attention due to their experiential, collaborative, and intensive nature, which certain tasks could be similar to real-world software development. This paper aims to discuss the idea of hackathons as an educational strategy for shaping SE students' soft skills in practice. Initially, we overview the existing literature on soft skills and hackathons in SE education. Then, we report preliminary empirical evidence from a seven-day hybrid hackathon involving 40 students. We assess how the hackathon experience promoted innovative and creative thinking, collaboration and teamwork, and knowledge application among participants through a structured questionnaire designed to evaluate students' self-awareness. Lastly, our findings and new directions are a\-nalyzed through the lens of Self-Determination Theory (SDT), which offers a  psychological lens to understand human behavior. This paper contributes to academia by advocating the potential of hackathons in SE education and proposing concrete plans for future research within SDT. For industry, our discussion has implications around developing soft skills in future SE professionals, thereby enhancing their employability and readiness in the software market.},
  booktitle = {Proceedings of the 37th IEEE Conference on Software Engineering Education and Training (CSEE&T)},
  pages = {},
  numpages = {},
  keywords = {Hackathon, Soft Skills, Self-Determination Theory, Experiential Learning},
  location = {Ottawa, Canada},
  note = {<font color="red">Accepted for publication.</font>},
  series = {CSEE&T'25}
}

In recent years, Software Engineering (SE) scholars and practitioners have emphasized the importance of integrating soft skills into SE education. However, teaching and learning soft skills are complex, as they cannot be acquired passively through raw knowledge acquisition. On the other hand, hackathons have attracted increasing attention due to their experiential, collaborative, and intensive nature, which certain tasks could be similar to real-world software development. This paper aims to discuss the idea of hackathons as an educational strategy for shaping SE students' soft skills in practice. Initially, we overview the existing literature on soft skills and hackathons in SE education. Then, we report preliminary empirical evidence from a seven-day hybrid hackathon involving 40 students. We assess how the hackathon experience promoted innovative and creative thinking, collaboration and teamwork, and knowledge application among participants through a structured questionnaire designed to evaluate students' self-awareness. Lastly, our findings and new directions are a­nalyzed through the lens of Self-Determination Theory (SDT), which offers a psychological lens to understand human behavior. This paper contributes to academia by advocating the potential of hackathons in SE education and proposing concrete plans for future research within SDT. For industry, our discussion has implications around developing soft skills in future SE professionals, thereby enhancing their employability and readiness in the software market.

@inproceedings{AraujoKPG25,
  author = {Araujo, Allysson Allex and Kalinowski, Marcos and Paixao, Matheus and Graziotin, Daniel},
  title = {Towards Emotionally Intelligent Software Engineers: Understanding Students' Self-Perceptions After a Cooperative Learning Experience},
  year = {2025},
  isbn = {},
  publisher = {IEEE},
  urlAuthor_version = {http://www.inf.puc-rio.br/~kalinowski/publications/AraujoKPG25.pdf},   
  doi = {},
  abstract = {[Background] Emotional Intelligence (EI) can impact Software Engineering (SE) outcomes through improved team communication, conflict resolution, and stress management. SE workers face increasing pressure to develop both technical and interpersonal skills, as modern software development emphasizes collaborative work and complex team interactions. Despite EI's documented importance in professional practice, SE education continues to prioritize technical knowledge over emotional and social competencies. [Objective] This paper analyzes SE students' self-perceptions of their EI after a two-month cooperative learning project, using Mayer and Salovey's four-ability model to examine how students handle emotions in collaborative development. [Method] We conducted a case study with 29 SE students organized into four squads within a project-based learning course, collecting data through questionnaires and focus groups that included brainwriting and sharing circles, then analyzing the data using descriptive statistics and open coding. [Results] Students demonstrated stronger abilities in managing their own emotions compared to interpreting others' emotional states. Despite limited formal EI training, they developed informal strategies for emotional management, including structured planning and peer support networks, which they connected to improved productivity and conflict resolution. [Conclusion] This study shows how SE students perceive EI in a collaborative learning context and provides evidence-based insights into the important role of emotional competencies in SE education.},
  booktitle = {Proceedings of the 18th IEEE/ACM International Conference on Cooperative and Human Aspects of Software Engineering},
  pages = {},
  numpages = {12},
  keywords = {Emotional Intelligence, Cooperative Learning, Project-based Learning, Software Engineering Education},
  location = {Ottawa, Canada},
  note = {<font color="red">Accepted for publication.</font>},
  series = {CHASE'25}
}

[Background] Emotional Intelligence (EI) can impact Software Engineering (SE) outcomes through improved team communication, conflict resolution, and stress management. SE workers face increasing pressure to develop both technical and interpersonal skills, as modern software development emphasizes collaborative work and complex team interactions. Despite EI's documented importance in professional practice, SE education continues to prioritize technical knowledge over emotional and social competencies. [Objective] This paper analyzes SE students' self-perceptions of their EI after a two-month cooperative learning project, using Mayer and Salovey's four-ability model to examine how students handle emotions in collaborative development. [Method] We conducted a case study with 29 SE students organized into four squads within a project-based learning course, collecting data through questionnaires and focus groups that included brainwriting and sharing circles, then analyzing the data using descriptive statistics and open coding. [Results] Students demonstrated stronger abilities in managing their own emotions compared to interpreting others' emotional states. Despite limited formal EI training, they developed informal strategies for emotional management, including structured planning and peer support networks, which they connected to improved productivity and conflict resolution. [Conclusion] This study shows how SE students perceive EI in a collaborative learning context and provides evidence-based insights into the important role of emotional competencies in SE education.

Generative AI (GenAI) introduces transformative challenges and opportunities to Intellectual Property (IP) processes in countries like Brazil, where existing laws, such as Copyright and IP Laws, do not explicitly account for AI-generated nuances. This paper explores the emerging idea behind GenAI’s impact and disruptive potential on the current Brazilian IP landscape. Using a qualitative approach, we apply McLuhan’s tetrad analysis, informed by Sociotechnical Theory, to identify potential enhancements, obsolescence, retrievals, and reversals that GenAI can bring to IP management in Brazil. Our contributions include advancing the understanding of the influence of GenAI on IP and offering preliminary insights for stakeholders to address emerging challenges and optimize opportunities in Brazil's evolving IP scenario.

@inproceedings{BarrosAGKKNB25,
  author = {Barros, Cauã Ferreira and Azevedo, Bruna Borges and Neto, Valdemar Vicente Graciano and Kassab, Mohamad and Kalinowski, Marcos and Nascimento, Hugo Alexandre D. and Bandeira, Michelle C. G. S. P.},
  title = {Large Language Model for Qualitative Research - A Systematic Mapping Study},
  year = {2025},
  isbn = {},
  publisher = {IEEE},
  urlAuthor_version = {http://www.inf.puc-rio.br/~kalinowski/publications/BarrosAGKKNB25.pdf}, 
  doi = {},
  abstract = {The exponential growth of text-based data in domains such as healthcare, education, and social sciences has outpaced the capacity of traditional qualitative analysis methods, which are time-intensive and prone to subjectivity. Large Language Models (LLMs), powered by advanced generative AI, have emerged as transformative tools capable of automating and enhancing qualitative analysis. This study systematically maps the literature on the use of LLMs for qualitative research, exploring their application contexts, configurations, methodologies, and evaluation metrics. Findings reveal that LLMs are utilized across diverse fields, demonstrating the potential to automate processes traditionally requiring extensive human input. However, challenges such as reliance on prompt engineering, occasional inaccuracies, and contextual limitations remain significant barriers. This research highlights opportunities for integrating LLMs with human expertise, improving model robustness, and refining evaluation methodologies. By synthesizing trends and identifying research gaps, this study aims to guide future innovations in the application of LLMs for qualitative analysis.},
  booktitle = {Proceedings of the 2nd IEEE/ACM International Workshop on Methodological Issues with Empirical Studies in Software Engineering},
  pages = {},
  numpages = {8},
  keywords = {Qualitative Research, Qualitative Analysis, Large Language Model, LLM, Software Engineering},
  location = {Ottawa, Canada},
  note = {<font color="red">Accepted for publication.</font>},
  series = {WSESE@ICSE '25}
}

The exponential growth of text-based data in domains such as healthcare, education, and social sciences has outpaced the capacity of traditional qualitative analysis methods, which are time-intensive and prone to subjectivity. Large Language Models (LLMs), powered by advanced generative AI, have emerged as transformative tools capable of automating and enhancing qualitative analysis. This study systematically maps the literature on the use of LLMs for qualitative research, exploring their application contexts, configurations, methodologies, and evaluation metrics. Findings reveal that LLMs are utilized across diverse fields, demonstrating the potential to automate processes traditionally requiring extensive human input. However, challenges such as reliance on prompt engineering, occasional inaccuracies, and contextual limitations remain significant barriers. This research highlights opportunities for integrating LLMs with human expertise, improving model robustness, and refining evaluation methodologies. By synthesizing trends and identifying research gaps, this study aims to guide future innovations in the application of LLMs for qualitative analysis.

@inproceedings{CostalongaNBFSK25,
  author = {Costalonga, Marcelo and Napole\~{a}o, Bianca Minetto and Baldassarre, Maria Teresa and Felizardo, Katia Romero and Steinmacher, Igor and Kalinowski, Marcos},
  title = {Can Machine Learning Support the Selection of Studies for Systematic Literature Review Updates?},
  year = {2025},
  isbn = {},
  publisher = {IEEE},
  urlAuthor_version = {http://www.inf.puc-rio.br/~kalinowski/publications/CostalongaNBFSK25.pdf}, 
  doi = {},
  abstract = {[Background] Systematic literature reviews (SLRs) are essential for synthesizing evidence in Software Engineering (SE), but keeping them up-to-date requires substantial effort. Study selection, one of the most labor-intensive steps, involves reviewing numerous studies and requires multiple reviewers to minimize bias and avoid loss of evidence. [Objective] This study aims to evaluate if Machine Learning (ML) text classification models can support reviewers in the study selection for SLR updates. [Method] We reproduce the study selection of an SLR update performed by three SE researchers. We trained two supervised ML models (Random Forest and Support Vector Machines) with different configurations using data from the original SLR. We calculated the study selection effectiveness of the ML models for the SLR update in terms of precision, recall, and F-measure. We also compared the performance of human-ML pairs with human-only pairs when selecting studies. [Results] The ML models achieved a modest F-score of 0.33, which is insufficient for reliable automation. However, we found that such models can reduce the study selection effort by 33.9% without loss of evidence (keeping a 100% recall). Our analysis also showed that the initial screening by pairs of human reviewers produces results that are much better aligned with the final SLR update result. [Conclusion] Based on our results, we conclude that although ML models can help reduce the effort involved in SLR updates, achieving rigorous and reliable outcomes still requires the expertise of experienced human reviewers for the initial screening phase.},
  booktitle = {Proceedings of the 2nd IEEE/ACM International Workshop on Methodological Issues with Empirical Studies in Software Engineering},
  pages = {},
  numpages = {8},
  keywords = {Systematic Review Automation, Selection of Studies, Machine Learning, Systematic Literature Review Update},
  location = {Ottawa, Canada},
  note = {<font color="red">Accepted for publication.</font>},
  series = {WSESE@ICSE '25}
}

[Background] Systematic literature reviews (SLRs) are essential for synthesizing evidence in Software Engineering (SE), but keeping them up-to-date requires substantial effort. Study selection, one of the most labor-intensive steps, involves reviewing numerous studies and requires multiple reviewers to minimize bias and avoid loss of evidence. [Objective] This study aims to evaluate if Machine Learning (ML) text classification models can support reviewers in the study selection for SLR updates. [Method] We reproduce the study selection of an SLR update performed by three SE researchers. We trained two supervised ML models (Random Forest and Support Vector Machines) with different configurations using data from the original SLR. We calculated the study selection effectiveness of the ML models for the SLR update in terms of precision, recall, and F-measure. We also compared the performance of human-ML pairs with human-only pairs when selecting studies. [Results] The ML models achieved a modest F-score of 0.33, which is insufficient for reliable automation. However, we found that such models can reduce the study selection effort by 33.9% without loss of evidence (keeping a 100% recall). Our analysis also showed that the initial screening by pairs of human reviewers produces results that are much better aligned with the final SLR update result. [Conclusion] Based on our results, we conclude that although ML models can help reduce the effort involved in SLR updates, achieving rigorous and reliable outcomes still requires the expertise of experienced human reviewers for the initial screening phase.

@inproceedings{GuenesTTBSK25,
  author = {Guenes, Paloma and Tomaz, Rafael and Trinkenreich, Bianca and Baldassarre, Maria Teresa and Storey, Margaret-Anne and Kalinowski, Marcos},
  title = {Impostor Phenomenon Among Software Engineers: Investigating Gender Differences and Well-Being},
  year = {2025},
  isbn = {},
  publisher = {IEEE},
  urlAuthor_version = {http://www.inf.puc-rio.br/~kalinowski/publications/GuenesTTBSK25.pdf}, 
  doi = {},
  abstract = {Research shows that more than half of software professionals experience the Impostor Phenomenon (IP), with a notably higher prevalence among women compared to men. IP can lead to mental health consequences, such as depression and burnout, which can significantly impact personal well-being and software professionals' productivity. This study investigates how IP manifests among software professionals across intersections of gender with race/ethnicity, marital status, number of children, age, and professional experience. Additionally, it examines the well-being of software professionals experiencing IP, providing insights into the interplay between these factors. We analyzed data collected through a theory-driven survey (n = 624) that used validated psychometric instruments to measure IP and well-being in software engineering professionals. We explored the prevalence of IP in the intersections of interest. Additionally, we applied bootstrapping to characterize well-being within our field and statistically tested whether professionals of different genders suffering from IP have lower well-being. The results show that IP occurs more frequently in women and that the prevalence is particularly high among black women as well as among single and childless women. Furthermore, regardless of gender, software engineering professionals suffering from IP have significantly lower well-being. Our findings indicate that effective IP mitigation strategies are needed to improve the well-being of software professionals. Mitigating IP would have particularly positive effects on the well-being of women, who are more frequently affected by IP.},
  booktitle = {Proceedings of the Sixth Workshop on Gender Equality, Diversity, and Inclusion in Software Engineering (GE@ICSE'25)},
  pages = {},
  numpages = {8},
  keywords = {impostor phenomenon, imposter syndrome, human aspects, perceived productivity, software engineering},
  location = {Ottawa, Canada},
  note = {<font color="red">Accepted for publication.</font>},
  series = {GE@ICSE'25}
}

Research shows that more than half of software professionals experience the Impostor Phenomenon (IP), with a notably higher prevalence among women compared to men. IP can lead to mental health consequences, such as depression and burnout, which can significantly impact personal well-being and software professionals' productivity. This study investigates how IP manifests among software professionals across intersections of gender with race/ethnicity, marital status, number of children, age, and professional experience. Additionally, it examines the well-being of software professionals experiencing IP, providing insights into the interplay between these factors. We analyzed data collected through a theory-driven survey (n = 624) that used validated psychometric instruments to measure IP and well-being in software engineering professionals. We explored the prevalence of IP in the intersections of interest. Additionally, we applied bootstrapping to characterize well-being within our field and statistically tested whether professionals of different genders suffering from IP have lower well-being. The results show that IP occurs more frequently in women and that the prevalence is particularly high among black women as well as among single and childless women. Furthermore, regardless of gender, software engineering professionals suffering from IP have significantly lower well-being. Our findings indicate that effective IP mitigation strategies are needed to improve the well-being of software professionals. Mitigating IP would have particularly positive effects on the well-being of women, who are more frequently affected by IP.

@InProceedings{KalinowskiRRBVBL25,
  author="Kalinowski, Marcos
  and Romao, Lucas
  and Rodrigues, Ariane
  and Barbosa, Clarissa
  and Villamizar, Hugo
  and Barbosa, Simone D.J.
  and Lopes, Helio",
  editor="Winkler, Dietmar
  and Ramler, Rudolf
  and Fischbach, Jannik
  and Bergsmann, Johannes",
  title="Experiences Applying Lean R&D in Industry-Academia Collaboration Projects",
  booktitle= {17th International Conference on Software Quality, Software Quality Days {SWQD} 2025, Munich, Germany, May 20-22},
  year="2025",
  publisher="Springer Nature Switzerland",
  address="Cham",
  pages="1--15",
  abstract="Lean R&D has been used at PUC-Rio to foster industry-academia collaboration in innovation projects across multiple sectors. This industrial experience paper describes recent experiences and evaluation results from applying Lean R&D in partnership with Petrobras in the oil and gas sector and Americanas in retail. The findings highlight Lean R&D's effectiveness in transforming ideas into meaningful business outcomes. Based on responses from 57 participants — including team members, managers, and sponsors — the assessment indicates that stakeholders find the structured phases of Lean R&D well-suited to innovation projects and endorse the approach. Although acknowledging that successful collaboration relies on various factors, this industrial experience positions Lean R&D as a promising framework for industry-academia projects focused on achieving rapid, impactful results for industry partners.",
  isbn="978-3-031-89277-6",
  doi="10.1007/978-3-031-89277-6_7",
  note="<font color=red>Industrial Experience Best Paper Award!</font>",
  urlAuthor_version = {http://www.inf.puc-rio.br/~kalinowski/publications/KalinowskiRRBVBL25.pdf}    
}

Lean R&D has been used at PUC-Rio to foster industry-academia collaboration in innovation projects across multiple sectors. This industrial experience paper describes recent experiences and evaluation results from applying Lean R&D in partnership with Petrobras in the oil and gas sector and Americanas in retail. The findings highlight Lean R&D's effectiveness in transforming ideas into meaningful business outcomes. Based on responses from 57 participants — including team members, managers, and sponsors — the assessment indicates that stakeholders find the structured phases of Lean R&D well-suited to innovation projects and endorse the approach. Although acknowledging that successful collaboration relies on various factors, this industrial experience positions Lean R&D as a promising framework for industry-academia projects focused on achieving rapid, impactful results for industry partners.

@inproceedings{PereiraKBBAM25,
  author = {Anrafel Fernandes Pereira and Marcos Kalinowski and Maria Teresa Baldassarre and Jürgen Börstler and Nauman bin Ali and Daniel Mendez},
  title = {Towards Lean Research Inception: Assessing Practical Relevance of Formulated Research Problems},
  year = {2025},
  isbn = {},
  publisher = {Association for Computing Machinery},
  urlAuthor_version = {http://www.inf.puc-rio.br/~kalinowski/publications/PereiraKBBAM25.pdf},  
  doi = {},
  abstract = {[Context] The lack of practical relevance in many Software Engineering (SE) research contributions is often rooted in oversimplified views of industrial practice, weak industry connections, and poorly defined research problems. Clear criteria for evaluating SE research problems can help align their value, feasibility, and applicability with industrial needs. [Goal] In this paper, we introduce the Lean Research Inception (LRI) framework, designed to support the formulation and assessment of practically relevant research problems in SE. We describe its initial evaluation strategy conducted in a workshop with a network of SE researchers experienced in industry-academia collaboration and report the evaluation of its three assessment criteria valuable, feasible, and applicable regarding their importance and completeness in assessing practical relevance. [Method] We applied LRI retroactively to a published research paper, engaging workshop participants in discussing and assessing the research problem by applying the proposed criteria using a semantic differential scale. Participants provided feedback on the criteria's importance and completeness, drawn from their own experiences in industry-academia collaboration. [Results] The findings reveal an overall agreement on the importance of the three criteria - valuable (83.3%), feasible (76.2%), and applicable (73.8\%) - for aligning research problems with industrial needs. Qualitative feedback suggested adjustments in terminology with a clearer distinction between feasible and applicable, and refinements for valuable by more clearly considering business value, ROI, and originality. [Conclusion] While LRI still constitutes ongoing research and requires further evaluation, our emerging results strengthen our confidence that the three criteria applied using the semantic differential scale can already help the community assess the practical relevance of SE research problems.},
  booktitle = {Proceedings of the 28th International Conference on Evaluation and Assessment in Software Engineering},
  pages = {1--6},
  numpages = {6},
  keywords = {Relevance Assessment, Research Problem Formulation, Practical Relevance},
  location = {Istanbul, Türkiye},
  note="<font color=red>Accepted for publication.</font>",
  series = {EASE '25}
}

[Context] The lack of practical relevance in many Software Engineering (SE) research contributions is often rooted in oversimplified views of industrial practice, weak industry connections, and poorly defined research problems. Clear criteria for evaluating SE research problems can help align their value, feasibility, and applicability with industrial needs. [Goal] In this paper, we introduce the Lean Research Inception (LRI) framework, designed to support the formulation and assessment of practically relevant research problems in SE. We describe its initial evaluation strategy conducted in a workshop with a network of SE researchers experienced in industry-academia collaboration and report the evaluation of its three assessment criteria valuable, feasible, and applicable regarding their importance and completeness in assessing practical relevance. [Method] We applied LRI retroactively to a published research paper, engaging workshop participants in discussing and assessing the research problem by applying the proposed criteria using a semantic differential scale. Participants provided feedback on the criteria's importance and completeness, drawn from their own experiences in industry-academia collaboration. [Results] The findings reveal an overall agreement on the importance of the three criteria - valuable (83.3%), feasible (76.2%), and applicable (73.8%) - for aligning research problems with industrial needs. Qualitative feedback suggested adjustments in terminology with a clearer distinction between feasible and applicable, and refinements for valuable by more clearly considering business value, ROI, and originality. [Conclusion] While LRI still constitutes ongoing research and requires further evaluation, our emerging results strengthen our confidence that the three criteria applied using the semantic differential scale can already help the community assess the practical relevance of SE research problems.

@inproceedings{TrinkenreichCHBKPTS25,
  author = {Bianca Trinkenreich and Fabio Calefato and Geir Hanssen and Kelly Blincoe and Marcos Kalinowski and Mauro Pezzè and Paolo Tell and Margaret-Anne Storey},
  title = {Get on the Train or be Left on the Station: Using LLMs for Software Engineering Research},
  year = {2025},
  isbn = {},
  publisher = {Association for Computing Machinery},
  urlAuthor_version = {http://www.inf.puc-rio.br/~kalinowski/publications/TrinkenreichCHBKPTS25.pdf},   
  doi = {},
  abstract = {The adoption of Large Language Models (LLMs) is not only transforming software engineering (SE) practice but is also poised to fundamentally disrupt how research is conducted in the field. While perspectives on this transformation range from viewing LLMs as mere productivity tools to considering them revolutionary forces, we argue that the SE research community must proactively engage with and shape the integration of LLMs into research practices, emphasizing human agency in this transformation. As LLMs rapidly become integral to SE research—both as tools that support investigations and as subjects of study—a human-centric perspective is essential. Ensuring human oversight and interpretability is necessary for upholding scientific rigor, fostering ethical responsibility, and driving  advancements in the field. Drawing from discussions at the 2nd Copenhagen Symposium on Human-Centered AI in SE, this position paper employs  McLuhan's Tetrad of Media Laws to analyze the impact of LLMs on SE research. Through this theoretical lens, we examine how LLMs enhance research capabilities through accelerated ideation and automated processes, make some traditional research practices obsolete, retrieve valuable aspects of historical research approaches, and risk reversal effects when taken to extremes. Our analysis reveals opportunities for innovation and potential pitfalls that require careful consideration. We conclude with a call to action for the SE research community to proactively harness the benefits of LLMs while developing frameworks and guidelines to mitigate their risks, to ensure continued rigor and impact of research in an AI-augmented future.},
  booktitle = {Proceedings of the 1st Workshop on Human-Centered AI for SE (Human AISE) held at the 33rd ACM International Conference on the Foundations of Software Engineering (FSE Companion ’25), June 23–28},
  pages = {},
  numpages = {5},
  keywords = {Generative AI, LLM, AI4SE, McLuhan’s Tetrad},
  location = { Trondheim, Norway},
  note="<font color=red>Accepted for publication.</font>",  
  series = {HumanAISE@FSE '25}  
}

The adoption of Large Language Models (LLMs) is not only transforming software engineering (SE) practice but is also poised to fundamentally disrupt how research is conducted in the field. While perspectives on this transformation range from viewing LLMs as mere productivity tools to considering them revolutionary forces, we argue that the SE research community must proactively engage with and shape the integration of LLMs into research practices, emphasizing human agency in this transformation. As LLMs rapidly become integral to SE research—both as tools that support investigations and as subjects of study—a human-centric perspective is essential. Ensuring human oversight and interpretability is necessary for upholding scientific rigor, fostering ethical responsibility, and driving advancements in the field. Drawing from discussions at the 2nd Copenhagen Symposium on Human-Centered AI in SE, this position paper employs McLuhan's Tetrad of Media Laws to analyze the impact of LLMs on SE research. Through this theoretical lens, we examine how LLMs enhance research capabilities through accelerated ideation and automated processes, make some traditional research practices obsolete, retrieve valuable aspects of historical research approaches, and risk reversal effects when taken to extremes. Our analysis reveals opportunities for innovation and potential pitfalls that require careful consideration. We conclude with a call to action for the SE research community to proactively harness the benefits of LLMs while developing frameworks and guidelines to mitigate their risks, to ensure continued rigor and impact of research in an AI-augmented future.

@inproceedings{XimenesAESK25,
  author = {Ximenes, Rodrigo and Alves, Antonio Pedro Santos and Escovedo, Tatiana and Spinola, Rodrigo and Kalinowski, Marcos},
  title = {Investigating Issues that Lead to Code Technical Debt in Machine Learning Systems},
  year = {2025},
  isbn = {},
  publisher = {IEEE},
  urlAuthor_version = {http://www.inf.puc-rio.br/~kalinowski/publications/XimenesAESK25.pdf},  
  doi = {},
  abstract = {[Context] Technical debt (TD) in machine learning (ML) systems, much like its counterpart in software engineering (SE), holds the potential to lead to future rework, posing risks to productivity, quality, and team morale. Despite growing attention to TD in SE, the understanding of ML-specific code-related TD remains underexplored. [Objective] This paper aims to identify and discuss the relevance of code-related issues that lead to TD in ML code throughout the ML workflow. [Method] The study first compiled a list of 34 potential issues contributing to TD in ML code by examining the phases of the ML workflow, their typical associated activities, and problem types. This list was refined through two focus group sessions involving nine experienced ML professionals, where each issue was assessed based on its occurrence contributing to TD in ML code and its relevance. [Results] The list of issues contributing to TD in the source code of ML systems was refined from 34 to 30, with 24 of these issues considered highly relevant. The \emph{data pre-processing} phase was the most critical, with 14 issues considered highly relevant. Shortcuts in code related to typical pre-processing tasks (\textit{e.g.}, handling missing values, outliers, inconsistencies, scaling, rebalancing, and feature selection) often result in ``patch fixes" rather than sustainable solutions, leading to the accumulation of TD and increasing maintenance costs. Relevant issues were also found in the  \emph{data collection}, \emph{model creation and training}, and \emph{model evaluation} phases. [Conclusion] We have made the final list of issues available to the community and believe it will help raise awareness about issues that need to be addressed throughout the ML workflow to reduce TD and improve the maintainability of ML code.},
  booktitle = {Proceedings of the IEEE/ACM 4th International Conference on AI Engineering - Software Engineering for AI},
  pages = {},
  numpages = {11},
  keywords = {Technical Debt, Machine Learning, Focus Group},
  location = {Ottawa, Canada},
  note = {<font color="red">Accepted for publication.</font>},  
  series = {CAIN '25}
}

[Context] Technical debt (TD) in machine learning (ML) systems, much like its counterpart in software engineering (SE), holds the potential to lead to future rework, posing risks to productivity, quality, and team morale. Despite growing attention to TD in SE, the understanding of ML-specific code-related TD remains underexplored. [Objective] This paper aims to identify and discuss the relevance of code-related issues that lead to TD in ML code throughout the ML workflow. [Method] The study first compiled a list of 34 potential issues contributing to TD in ML code by examining the phases of the ML workflow, their typical associated activities, and problem types. This list was refined through two focus group sessions involving nine experienced ML professionals, where each issue was assessed based on its occurrence contributing to TD in ML code and its relevance. [Results] The list of issues contributing to TD in the source code of ML systems was refined from 34 to 30, with 24 of these issues considered highly relevant. The \emphdata pre-processing phase was the most critical, with 14 issues considered highly relevant. Shortcuts in code related to typical pre-processing tasks (e.g., handling missing values, outliers, inconsistencies, scaling, rebalancing, and feature selection) often result in ``patch fixes" rather than sustainable solutions, leading to the accumulation of TD and increasing maintenance costs. Relevant issues were also found in the \emphdata collection, \emphmodel creation and training, and \emphmodel evaluation phases. [Conclusion] We have made the final list of issues available to the community and believe it will help raise awareness about issues that need to be addressed throughout the ML workflow to reduce TD and improve the maintainability of ML code.

@ARTICLE{CarvalhoEtAl24,
  author={Carvalho, Luiz and Colanzi, Thelma Elita and Assunção, Wesley K. G. and Garcia, Alessandro and Pereira, Juliana Alves and Kalinowski, Marcos and de Mello, Rafael Maiani and de Lima, Maria Julia and Lucena, Carlos},
  journal={IEEE Transactions on Software Engineering}, 
  title={On the Usefulness of Automatically Generated Microservice Architectures}, 
  abstract={The modernization of monolithic legacy systems with microservices has been a trend in recent years. As part of this modernization, identifying microservice candidates starting from legacy code is challenging, as maintainers may consider many criteria simultaneously. Multi-objective search-based approaches represent a promising state-of-the-art solution to support this decision-making process. However, the rationale to adopt each microservice candidate automatically identified by these approaches is poorly investigated in industrial cases. Furthermore, studies with these approaches have not carefully investigated how maintainers reason and make decisions when designing microservice architectures from legacy systems. To address this gap, we conducted an on-site case study with maintainers of an industrial legacy system to investigate the usefulness of automatically generated microservice architectures. We analyze design decisions pointed out by the maintainers when reasoning about microservice candidates using several criteria at the same time. Our study is the first to assess a search-based approach involving actual maintainers conceiving microservice architectures in an industrial setting. Therefore, firstly, we considered individual evaluation of microservice candidates to understand the rationale for identifying a service. Secondly, we conducted a focus group study with maintainers with the goal of investigating design decisions at an architectural level. The results show that: (i) the automated approach is able to identify useful microservices; (ii) the criteria observed by previous studies are, in fact, considered by maintainers; and (iii) the maintainer profiles, i.e., the preferred granularity for microservice, highly affect design decisions. Finally, we observed the maintainers needed little effort in adjusting the automatically identified microservices to make them adoptable. In addition to indicating a promising potential of search-based approaches to generate microservice architectures, our findings highlight the need for: (i) interactive and/or customizable approaches that enable maintainers to include their preferences during the search process, and (ii) flexible or automated selection of criteria that fits the scenario in which the modernization is taking place.},
  year={2024},
  volume={50},
  number={3},
  pages={651-667},
  keywords={Microservice architectures;Aging;Computer architecture;Decision making;Codes;Source coding;Interviews;Microservice architecture;search-based software engineering;microservice identification;legacy system;qualitative study},
  doi={10.1109/TSE.2024.3361209},
  urlAuthor_version = {http://www.inf.puc-rio.br/~kalinowski/publications/CarvalhoEtAl24.pdf}
}

The modernization of monolithic legacy systems with microservices has been a trend in recent years. As part of this modernization, identifying microservice candidates starting from legacy code is challenging, as maintainers may consider many criteria simultaneously. Multi-objective search-based approaches represent a promising state-of-the-art solution to support this decision-making process. However, the rationale to adopt each microservice candidate automatically identified by these approaches is poorly investigated in industrial cases. Furthermore, studies with these approaches have not carefully investigated how maintainers reason and make decisions when designing microservice architectures from legacy systems. To address this gap, we conducted an on-site case study with maintainers of an industrial legacy system to investigate the usefulness of automatically generated microservice architectures. We analyze design decisions pointed out by the maintainers when reasoning about microservice candidates using several criteria at the same time. Our study is the first to assess a search-based approach involving actual maintainers conceiving microservice architectures in an industrial setting. Therefore, firstly, we considered individual evaluation of microservice candidates to understand the rationale for identifying a service. Secondly, we conducted a focus group study with maintainers with the goal of investigating design decisions at an architectural level. The results show that: (i) the automated approach is able to identify useful microservices; (ii) the criteria observed by previous studies are, in fact, considered by maintainers; and (iii) the maintainer profiles, i.e., the preferred granularity for microservice, highly affect design decisions. Finally, we observed the maintainers needed little effort in adjusting the automatically identified microservices to make them adoptable. In addition to indicating a promising potential of search-based approaches to generate microservice architectures, our findings highlight the need for: (i) interactive and/or customizable approaches that enable maintainers to include their preferences during the search process, and (ii) flexible or automated selection of criteria that fits the scenario in which the modernization is taking place.

@article{SetubalCKA24,
  title={Investigating the Online Recruitment and Selection Journey of Novice Software Engineers: Anti-patterns and Recommendations}, 
  author={Miguel Setúbal and Tayana Conte and Marcos Kalinowski and Allysson Allex Araújo},
  journal={Empirical Software Engineering},
  volume={29},
  number={125},
  pages={1-30},
  year={2024},
  publisher={Springer},
  doi={10.1007/s10664-024-10498-w},
  urlAuthor_version = {http://www.inf.puc-rio.br/~kalinowski/publications/SetubalCKA24.pdf},
  abstract = {The growing software development market has increased the demand for qualified professionals in Software Engineering (SE). To this end, companies must enhance their Recruitment and Selection (R&S) processes to maintain high-quality teams, including opening opportunities for beginners, such as trainees and interns. However, given the various judgments and sociotechnical factors involved, this complex process of R&S poses a challenge for recent graduates seeking to enter the market. This paper aims to identify a set of anti-patterns and recommendations for early career SE professionals concerning R&S processes. Under an exploratory and qualitative methodological approach, we conducted six online Focus Groups with 18 recruiters with experience in R&S in the software industry. After completing our qualitative analysis, we identified 12 anti-patterns and 31 actionable recommendations regarding the hiring process focused on entry-level SE professionals. The identified anti-patterns encompass behavioral and technical dimensions innate to R&S processes. These findings provide a rich opportunity for reflection in the SE industry and offer valuable guidance for early-career candidates and organizations. From an academic perspective, this work also raises awareness of the intersection of Human Resources and SE, an area with considerable potential to be expanded in the context of cooperative and human aspects of SE.}
}

The growing software development market has increased the demand for qualified professionals in Software Engineering (SE). To this end, companies must enhance their Recruitment and Selection (R&S) processes to maintain high-quality teams, including opening opportunities for beginners, such as trainees and interns. However, given the various judgments and sociotechnical factors involved, this complex process of R&S poses a challenge for recent graduates seeking to enter the market. This paper aims to identify a set of anti-patterns and recommendations for early career SE professionals concerning R&S processes. Under an exploratory and qualitative methodological approach, we conducted six online Focus Groups with 18 recruiters with experience in R&S in the software industry. After completing our qualitative analysis, we identified 12 anti-patterns and 31 actionable recommendations regarding the hiring process focused on entry-level SE professionals. The identified anti-patterns encompass behavioral and technical dimensions innate to R&S processes. These findings provide a rich opportunity for reflection in the SE industry and offer valuable guidance for early-career candidates and organizations. From an academic perspective, this work also raises awareness of the intersection of Human Resources and SE, an area with considerable potential to be expanded in the context of cooperative and human aspects of SE.

@article{VillamizarKLM24,
  title = {Identifying concerns when specifying machine learning-enabled systems: A perspective-based approach},
  journal = {Journal of Systems and Software},
  volume = {213},
  pages = {112053},
  year = {2024},
  issn = {0164-1212},
  doi = {https://doi.org/10.1016/j.jss.2024.112053},
  urlAuthor_version = {http://www.inf.puc-rio.br/~kalinowski/publications/VillamizarKLM24.pdf},
  author = {Hugo Villamizar and Marcos Kalinowski and Hélio Lopes and Daniel Mendez},
  keywords = {Requirements engineering, Machine learning-enabled systems, Technology transfer, Case study},
  abstract = {Engineering successful machine learning (ML)-enabled systems poses various challenges from both a theoretical and a practical side. Among those challenges are how to effectively address unrealistic expectations of ML capabilities from customers, managers and even other team members, and how to connect business value to engineering and data science activities composed by interdisciplinary teams. In this paper, we present PerSpecML, a perspective-based approach for specifying ML-enabled systems that helps practitioners identify which attributes, including ML and non-ML components, are important to contribute to the overall system’s quality. The approach involves analyzing 60 concerns related to 28 tasks that practitioners typically face in ML projects, grouping them into five perspectives: system objectives, user experience, infrastructure, model, and data. Together, these perspectives serve to mediate the communication between business owners, domain experts, designers, software and ML engineers, and data scientists. The creation of PerSpecML involved a series of formative evaluations conducted in different contexts: (i) in academia, (ii) with industry representatives, and (iii) in two real industrial case studies. As a result of the diverse validations and continuous improvements, PerSpecML stands as a promising approach, poised to positively impact the specification of ML-enabled systems, particularly helping to reveal key components that would have been otherwise missed without using PerSpecML. Editor’s note: Open Science material was validated by the Journal of Systems and Software Open Science Board.}
}

Engineering successful machine learning (ML)-enabled systems poses various challenges from both a theoretical and a practical side. Among those challenges are how to effectively address unrealistic expectations of ML capabilities from customers, managers and even other team members, and how to connect business value to engineering and data science activities composed by interdisciplinary teams. In this paper, we present PerSpecML, a perspective-based approach for specifying ML-enabled systems that helps practitioners identify which attributes, including ML and non-ML components, are important to contribute to the overall system’s quality. The approach involves analyzing 60 concerns related to 28 tasks that practitioners typically face in ML projects, grouping them into five perspectives: system objectives, user experience, infrastructure, model, and data. Together, these perspectives serve to mediate the communication between business owners, domain experts, designers, software and ML engineers, and data scientists. The creation of PerSpecML involved a series of formative evaluations conducted in different contexts: (i) in academia, (ii) with industry representatives, and (iii) in two real industrial case studies. As a result of the diverse validations and continuous improvements, PerSpecML stands as a promising approach, poised to positively impact the specification of ML-enabled systems, particularly helping to reveal key components that would have been otherwise missed without using PerSpecML. Editor’s note: Open Science material was validated by the Journal of Systems and Software Open Science Board.

@inproceedings{AlvesKMVAEL24,
  author = {Antonio Alves and Marcos Kalinowski and Daniel Mendez and Hugo Villamizar and Kelly Azevedo and Tatiana Escovedo and Helio Lopes},
  title = {Industrial Practices of Requirements Engineering for ML-Enabled Systems in Brazil},
  abstract = {[Context] In Brazil, 41% of companies use machine learning (ML) to some extent. However, several challenges have been reported when engineering ML-enabled systems, including unrealistic customer expectations and vagueness in ML problem specifications. Literature suggests that Requirements Engineering (RE) practices and tools may help to alleviate these issues, yet there is insufficient understanding of RE’s practical application and its perception among practitioners. [Goal] This study aims to investigate the application of RE in developing ML-enabled systems in Brazil, creating an overview of current practices, perceptions, and problems in the Brazilian industry. [Method] To this end, we extracted and analyzed data from an international survey focused on ML-enabled systems, concentrating specifically on responses from practitioners based in Brazil. We analyzed the cluster of RE-related answers gathered from 72 practitioners involved in data-driven projects. We conducted quantitative statistical analyses on contemporary practices using bootstrapping with confidence intervals and qualitative studies on the reported problems involving open and axial coding procedures. [Results] Our findings highlight distinct aspects of RE implementation in ML projects in Brazil. For instance, (i) RErelated tasks are predominantly conducted by data scientists; (ii) the most common techniques for eliciting requirements are interviews and workshop meetings; (iii) there is a prevalence of interactive notebooks in requirements documentation; (iv) practitioners report problems that include a poor understanding of the problem to solve and the business domain, low customer engagement, and difficulties managing stakeholders expectations. [Conclusion] These results provide an understanding of RE-related practices in the Brazilian ML industry, helping to guide research and initiatives toward improving the maturity of RE for ML-enabled systems.},
  booktitle = {Anais do XXXVIII Simpósio Brasileiro de Engenharia de Software},
  note={<font color="red">SBES 2024 Distinguished Paper Award!</font>},  
  location = {Curitiba/PR},
  year = {2024},
  keywords = {Requirements Engineering, Machine Learning, Survey, Brazil},
  issn = {0000-0000},
  pages = {224--233},
  publisher = {SBC},
  doi = {10.5753/sbes.2024.3371},
  urlAuthor_version = {http://www.inf.puc-rio.br/~kalinowski/publications/AlvesKMVAEL24.pdf}
}

[Context] In Brazil, 41% of companies use machine learning (ML) to some extent. However, several challenges have been reported when engineering ML-enabled systems, including unrealistic customer expectations and vagueness in ML problem specifications. Literature suggests that Requirements Engineering (RE) practices and tools may help to alleviate these issues, yet there is insufficient understanding of RE’s practical application and its perception among practitioners. [Goal] This study aims to investigate the application of RE in developing ML-enabled systems in Brazil, creating an overview of current practices, perceptions, and problems in the Brazilian industry. [Method] To this end, we extracted and analyzed data from an international survey focused on ML-enabled systems, concentrating specifically on responses from practitioners based in Brazil. We analyzed the cluster of RE-related answers gathered from 72 practitioners involved in data-driven projects. We conducted quantitative statistical analyses on contemporary practices using bootstrapping with confidence intervals and qualitative studies on the reported problems involving open and axial coding procedures. [Results] Our findings highlight distinct aspects of RE implementation in ML projects in Brazil. For instance, (i) RErelated tasks are predominantly conducted by data scientists; (ii) the most common techniques for eliciting requirements are interviews and workshop meetings; (iii) there is a prevalence of interactive notebooks in requirements documentation; (iv) practitioners report problems that include a poor understanding of the problem to solve and the business domain, low customer engagement, and difficulties managing stakeholders expectations. [Conclusion] These results provide an understanding of RE-related practices in the Brazilian ML industry, helping to guide research and initiatives toward improving the maturity of RE for ML-enabled systems.

@inproceedings{AlvesKM24,
  author = {Alves, Antonio Pedro Santos and Kalinowski, Marcos and M\'{e}ndez, Daniel},
  title = {Requirements Engineering for ML-Enabled Systems: Status Quo and Problems},
  year = {2024},
  isbn = {9798400717772},
  publisher = {Association for Computing Machinery},
  urlAuthor_version = {http://www.inf.puc-rio.br/~kalinowski/publications/AlvesKM24.pdf},
  doi = {10.1145/3701625.3701697},
  abstract = {Machine Learning (ML) systems are increasingly common in companies seeking to improve products and processes. While literature suggests that Requirements Engineering (RE) can address challenges in ML-enabled systems, most empirical studies are isolated and lack generalizability. The goal of this dissertation is to enhance the empirical evidence on the intersection of RE and ML-enabled systems. For this purpose, we conducted an international survey with 188 respondents from 25 countries, using statistical and qualitative analyses to explore RE practices and challenges in ML projects. Our key findings include: (i) project leaders and data scientists primarily handle RE activities, (ii) interactive Notebooks are the dominant documentation format, (iii) data quality, model reliability, and explainability are the main non-functional requirements, (iv) challenges when developing such systems include managing customer expectations and aligning requirements with data, and (v) the main problems practitioners face are related to lack of business domain understanding, unclear goals and requirements, and low customer engagement. These results give us a wider picture of the adopted practices and the challenges in industrial scenarios. We put forward the need to adapt further and disseminate RE-related practices for engineering high-quality ML-enabled systems.},
  booktitle = {Proceedings of the XXIII Brazilian Symposium on Software Quality},
  pages = {697–699},
  numpages = {3},
  keywords = {Requirements Engineering, Machine Learning, Survey},
  location = {Salvador, Brazil},
  note = {<font color="red">Summary for the "Second Best Brazilian Software Quality MS Dissertation Award", received at SBQS 2024. Student: Antonio Pedro Santos Alves, Advisors: Marcos Kalinowski and Daniel Mendez.</font>},
  series = {SBQS '24}
}

Machine Learning (ML) systems are increasingly common in companies seeking to improve products and processes. While literature suggests that Requirements Engineering (RE) can address challenges in ML-enabled systems, most empirical studies are isolated and lack generalizability. The goal of this dissertation is to enhance the empirical evidence on the intersection of RE and ML-enabled systems. For this purpose, we conducted an international survey with 188 respondents from 25 countries, using statistical and qualitative analyses to explore RE practices and challenges in ML projects. Our key findings include: (i) project leaders and data scientists primarily handle RE activities, (ii) interactive Notebooks are the dominant documentation format, (iii) data quality, model reliability, and explainability are the main non-functional requirements, (iv) challenges when developing such systems include managing customer expectations and aligning requirements with data, and (v) the main problems practitioners face are related to lack of business domain understanding, unclear goals and requirements, and low customer engagement. These results give us a wider picture of the adopted practices and the challenges in industrial scenarios. We put forward the need to adapt further and disseminate RE-related practices for engineering high-quality ML-enabled systems.

@inproceedings{AlvesAAAK24,
  author    = {Antonio Pedro Santos Alves and Julia Araújo and Marina Araújo and Wallace Albertini and Marcos Kalinowski},
  title     = {Uma Análise da Necessidade de um Modelo de Referência MPS para Inteligência Artificial},
  abstract  = {O uso cada vez mais comum da Inteligência Artificial (IA) como ferramenta potencializadora de novos negócios, traz na sua incorporação em sistemas desafios únicos. Por anos, o Modelo de Referência MPS-SW apoiou a construção e evolução de sistemas de software de qualidade, contudo, o mesmo não contempla particularidades de soluções de IA. Neste artigo realizamos uma reflexão com base na adaptação da ISO 5338 feita em cima da ISO 12207 para sistemas de IA. Nesta adaptação, 23 processos foram modificados e três novos processos foram incluídos para melhor atender ao domínio de IA. Entendemos que o trabalho realizado no contexto da ISO 5338 possa apoiar a concepção de um modelo de referência MPS para IA.},
  booktitle = {Workshop Anual de Melhoria de Processos (WAMPS 2024), Salvador, Brazil},
  pages     = {1-5},
  year      = {2024},
  urlAuthor_version = {http://www.inf.puc-rio.br/~kalinowski/publications/AlvesAAAK24.pdf},
  doi       = {},
}

O uso cada vez mais comum da Inteligência Artificial (IA) como ferramenta potencializadora de novos negócios, traz na sua incorporação em sistemas desafios únicos. Por anos, o Modelo de Referência MPS-SW apoiou a construção e evolução de sistemas de software de qualidade, contudo, o mesmo não contempla particularidades de soluções de IA. Neste artigo realizamos uma reflexão com base na adaptação da ISO 5338 feita em cima da ISO 12207 para sistemas de IA. Nesta adaptação, 23 processos foram modificados e três novos processos foram incluídos para melhor atender ao domínio de IA. Entendemos que o trabalho realizado no contexto da ISO 5338 possa apoiar a concepção de um modelo de referência MPS para IA.

@inproceedings{AntunesLATK24,
  author = {Fabio Antunes and Maria Lima and Marco Araújo and Davide Taibi and Marcos Kalinowski},
  title = {Investigating Benefits and Limitations of Migrating to a Micro-Frontends Architecture},
  abstract = {[Context] The adoption of micro-frontends architectures has gained traction as a promising approach to enhance modularity, scalability, and maintainability of web applications. [Goal] The primary aim of this research is to investigate the benefits and limitations of migrating a real-world application to a micro-frontends architecture from the perspective of the developers. [Method] Based on the action research approach, after diagnosis and planning, we applied an intervention of migrating the target web application to a micro-frontends architecture. Thereafter, the migration was evaluated in a workshop involving the remaining developers responsible for maintaining the application. During the workshop, these developers were presented with the migrated architecture, conducted a simple maintenance task, discussed benefits and limitations in a focus group to gather insights, and answered a questionnaire on the acceptance of the technology. [Results] Developers' perceptions gathered during the focus group reinforce the benefits and limitations reported in the literature. Key benefits included enhanced flexibility in technology choices, scalability of development teams, and gradual migration of technologies. However, the increased complexity of the architecture raised concerns among developers, particularly in dependency and environment management, debugging, and integration testing. [Conclusions] While micro-frontends represent a promising technology, unresolved issues still limit their broader applicability. Developers generally perceived the architecture as useful and moderately easy to use but hesitated to adopt it.},
  booktitle = {Anais do XXXVIII Simpósio Brasileiro de Engenharia de Software},
  location = {Curitiba/PR},
  year = {2024},
  keywords = {},
  issn = {0000-0000},
  pages = {103--113},
  publisher = {SBC},
  doi = {10.5753/sbes.2024.3303},
  urlAuthor_version = {http://www.inf.puc-rio.br/~kalinowski/publications/AntunesLATK24.pdf}
}

[Context] The adoption of micro-frontends architectures has gained traction as a promising approach to enhance modularity, scalability, and maintainability of web applications. [Goal] The primary aim of this research is to investigate the benefits and limitations of migrating a real-world application to a micro-frontends architecture from the perspective of the developers. [Method] Based on the action research approach, after diagnosis and planning, we applied an intervention of migrating the target web application to a micro-frontends architecture. Thereafter, the migration was evaluated in a workshop involving the remaining developers responsible for maintaining the application. During the workshop, these developers were presented with the migrated architecture, conducted a simple maintenance task, discussed benefits and limitations in a focus group to gather insights, and answered a questionnaire on the acceptance of the technology. [Results] Developers' perceptions gathered during the focus group reinforce the benefits and limitations reported in the literature. Key benefits included enhanced flexibility in technology choices, scalability of development teams, and gradual migration of technologies. However, the increased complexity of the architecture raised concerns among developers, particularly in dependency and environment management, debugging, and integration testing. [Conclusions] While micro-frontends represent a promising technology, unresolved issues still limit their broader applicability. Developers generally perceived the architecture as useful and moderately easy to use but hesitated to adopt it.

@inproceedings{AraujoKB24,
  author = {Araujo, Allysson Allex and Kalinowski, Marcos and Baldassarre, Maria Teresa},
  title = {Can participation in a hackathon impact the motivation of software engineering students? A preliminary case study analysis},
  year = {2024},
  isbn = {9798400705335},
  publisher = {Association for Computing Machinery},
  urlAuthor_version = {http://www.inf.puc-rio.br/~kalinowski/publications/AraujoKB24.pdf},   
  doi = {10.1145/3641822.3641876},
  abstract = {[Background] Hackathons are increasingly gaining prominence in Software Engineering (SE) education, lauded for their ability to elevate students' skill sets. [Objective] This paper investigates whether hackathons can impact the motivation of SE students. [Method] We conducted an evaluative case study assessing students' motivations before and after a hackathon, combining quantitative analysis using the Academic Motivation Scale (AMS) and qualitative coding of open-ended responses. [Results] Pre-hackathon findings reveal a diverse range of motivations with an overall acceptance, while post-hackathon responses highlight no statistically significant shift in participants' perceptions. Qualitative findings uncovered themes related to networking, team dynamics, and skill development. From a practical perspective, our findings highlight the potential of hackathons to impact participants' motivation. [Conclusion] While our study enhances the comprehension of hackathons as a motivational tool, it also underscores the need for further exploration of psychometric dimensions in SE educational research.},
  booktitle = {Proceedings of the 2024 IEEE/ACM 17th International Conference on Cooperative and Human Aspects of Software Engineering},
  pages = {103–108},
  numpages = {6},
  keywords = {hackathon, academic motivation, psychometrics, education},
  location = {Lisbon, Portugal},
  series = {CHASE '24}
}

[Background] Hackathons are increasingly gaining prominence in Software Engineering (SE) education, lauded for their ability to elevate students' skill sets. [Objective] This paper investigates whether hackathons can impact the motivation of SE students. [Method] We conducted an evaluative case study assessing students' motivations before and after a hackathon, combining quantitative analysis using the Academic Motivation Scale (AMS) and qualitative coding of open-ended responses. [Results] Pre-hackathon findings reveal a diverse range of motivations with an overall acceptance, while post-hackathon responses highlight no statistically significant shift in participants' perceptions. Qualitative findings uncovered themes related to networking, team dynamics, and skill development. From a practical perspective, our findings highlight the potential of hackathons to impact participants' motivation. [Conclusion] While our study enhances the comprehension of hackathons as a motivational tool, it also underscores the need for further exploration of psychometric dimensions in SE educational research.

@inproceedings{AraujoKG24,
  title={Understanding and measuring software engineer behavior: What can we learn from the behavioral sciences?}, 
  author={Allysson Allex Araújo and Marcos Kalinowski and Daniel Graziotin},
  year={2024},
  booktitle = {IX Workshop sobre Aspectos Sociais, Humanos e Econômicos de Software (CSBC 2024 - WASHES 2024), Brasília, Brazil},
  abstract = {This paper explores the intricate challenge of understanding and measuring software engineer behavior. More specifically, we revolve around a central question: How can we enhance our understanding of software engineer behavior? Grounded in the nuanced complexities addressed within Behavioral Software Engineering (BSE), we advocate for holistic methods that integrate quantitative measures, such as psychometric instruments, and qualitative data from diverse sources. Furthermore, we delve into the relevance of this challenge within national and international contexts, highlighting the increasing interest in understanding software engineer behavior. Real-world initiatives and academic endeavors are also examined to underscore the potential for advancing this research agenda and, consequently, refining software engineering practices based on behavioral aspects. Lastly, this paper addresses different ways to evaluate the progress of this challenge by leveraging methodological skills derived from behavioral sciences, ultimately contributing to a deeper understanding of software engineer behavior and software engineering practices.},
  pages     = {1-6},
  urlAuthor_version = {http://www.inf.puc-rio.br/~kalinowski/publications/AraujoKG24.pdf},
  doi = {},
}

This paper explores the intricate challenge of understanding and measuring software engineer behavior. More specifically, we revolve around a central question: How can we enhance our understanding of software engineer behavior? Grounded in the nuanced complexities addressed within Behavioral Software Engineering (BSE), we advocate for holistic methods that integrate quantitative measures, such as psychometric instruments, and qualitative data from diverse sources. Furthermore, we delve into the relevance of this challenge within national and international contexts, highlighting the increasing interest in understanding software engineer behavior. Real-world initiatives and academic endeavors are also examined to underscore the potential for advancing this research agenda and, consequently, refining software engineering practices based on behavioral aspects. Lastly, this paper addresses different ways to evaluate the progress of this challenge by leveraging methodological skills derived from behavioral sciences, ultimately contributing to a deeper understanding of software engineer behavior and software engineering practices.

@inproceedings{AraujoSDBK24,
  author = {Allysson Araújo and Joel Sousa and Vicente Duarte and Thiago Bessa and Marcos Kalinowski},
  title = {Caminhos do Interior: Connecting Information Systems, Tourism, and Regional Development in Brazil's Inland},
  booktitle = {Proceedings of the 20th Brazilian Symposium on Information Systems},
  abstract = {This paper introduces the "Caminhos do Interior", a proposal of web-based solution aimed at promoting regional development through tourism in the lesser-explored areas of Brazil. Drawing from an interdisciplinary and sociotechnical perspective, this project merges concepts of under tourism and city branding to curate and spotlight tourist attractions. We provide a web-based high-fidelity prototype showcasing regional attractions, city profiles, and a categorization of tourist activities. With potential impacts spanning academia, industry, and policy, our initiative contributes to the literature and practice by investigating the role of Information Systems (IS) in promoting regional growth and enriching the understanding of the interplay between IS, tourism, and socioeconomic development. Furthermore, we intend to offer practical and socially aware insights by demonstrating the role of an IS in attracting tourists, investors, entrepreneurs, and strengthening regional economies.},
  location = {Juiz de Fora/MG},
  series = {SBSI '24},
  year = {2024},
  keywords = {},
  publisher = {SBC},
  doi = {10.5753/sbsi_estendido.2024.238676},
  urlAuthor_version = {http://www.inf.puc-rio.br/~kalinowski/publications/AraujoSDBK24.pdf}
}

This paper introduces the "Caminhos do Interior", a proposal of web-based solution aimed at promoting regional development through tourism in the lesser-explored areas of Brazil. Drawing from an interdisciplinary and sociotechnical perspective, this project merges concepts of under tourism and city branding to curate and spotlight tourist attractions. We provide a web-based high-fidelity prototype showcasing regional attractions, city profiles, and a categorization of tourist activities. With potential impacts spanning academia, industry, and policy, our initiative contributes to the literature and practice by investigating the role of Information Systems (IS) in promoting regional growth and enriching the understanding of the interplay between IS, tourism, and socioeconomic development. Furthermore, we intend to offer practical and socially aware insights by demonstrating the role of an IS in attracting tourists, investors, entrepreneurs, and strengthening regional economies.

@inproceedings{AraujoKEC24,
  author       = {Gabriel Araujo and
                  Marcos Kalinowski and
                  Markus Endler and
                  Fabio Calefato},
  editor       = {Joaquim Filipe and
                  Michal Smialek and
                  Alexander Brodsky and
                  Slimane Hammoudi},
  title        = {Professional Insights into Benefits and Limitations of Implementing
                  MLOps Principles},
  booktitle    = {Proceedings of the 26th International Conference on Enterprise Information
                  Systems, {ICEIS} 2024, Angers, France, April 28-30, 2024, Volume 2},
  pages        = {305--312},
  publisher    = {{SCITEPRESS}},
  year         = {2024},
  doi          = {10.5220/0012741100003690},
  urlAuthor_version = {http://www.inf.puc-rio.br/~kalinowski/publications/AraujoKEC24.pdf}  
}

@incollection{AvgeriouAKM24,
  author = {Paris Avgeriou and Nauman bin Ali and Marcos Kalinowski and Daniel Mendez},
  title = {Designing a Syllabus for a Course on Empirical Software Engineering},
  editor = {Daniel Mendez and Paris Avgeriou and Marcos Kalinowski and Nauman bin Ali},
  booktitle = {Handbook on Teaching Empirical Software Engineering},
  publisher = {Springer},
  year = {2024},
  pages = {13-28},
  chapter = {Designing a Syllabus for a Course on Empirical Software Engineering},
  urlAuthor_version = {http://www.inf.puc-rio.br/~kalinowski/publications/AvgeriouAKM24.pdf},
  doi = {10.1007/978-3-031-71769-7_2}
}

@inproceedings{BaldassarreGKR24,
  author = {Baldassarre, Maria Teresa and Gigante, Domenico and Kalinowski, Marcos and Ragone, Azzurra},
  title = {POLARIS: A Framework to Guide the Development of Trustworthy AI Systems},
  year = {2024},
  isbn = {9798400705915},
  publisher = {Association for Computing Machinery},
  urlAuthor_version = {http://www.inf.puc-rio.br/~kalinowski/publications/BaldassarreGKR24.pdf},  
  doi = {10.1145/3644815.3644947},
  abstract = {In the ever-expanding landscape of Artificial Intelligence (AI), where innovation thrives and new products and services are continuously being delivered, ensuring that AI systems are designed and developed responsibly throughout their entire lifecycle is crucial. To this end, several AI ethics principles and guidelines have been issued to which AI systems should conform. Nevertheless, relying solely on high-level AI ethics principles is far from sufficient to ensure the responsible engineering of AI systems. In this field, AI professionals often navigate by sight. Indeed, while recommendations promoting Trustworthy AI (TAI) exist, they are often high-level statements difficult to translate into concrete implementation strategies. Currently, there is a significant gap between high-level AI ethics principles and low-level concrete practices for AI professionals. To address this challenge, our work presents an experience report where we develop a novel holistic framework for Trustworthy AI --- designed to bridge the gap between theory and practice --- and report insights from its application in an industrial case study. The framework builds up from the results of a systematic review of the state of the practice as well as a survey and think-aloud interviews with 34 AI practitioners. The framework, unlike most of the ones in literature, is designed to provide actionable guidelines and tools to support different types of stakeholders throughout the entire Software Development Life Cycle (SDLC). Our goal is to empower AI professionals to confidently navigate the ethical dimensions of TAI through practical insights, ensuring that the vast potential of AI is exploited responsibly for the benefit of society as a whole.},
  booktitle = {Proceedings of the IEEE/ACM 3rd International Conference on AI Engineering - Software Engineering for AI},
  pages = {200–210},
  numpages = {11},
  keywords = {artificial intelligence, software engineering, trustworthy AI, knowledge base, framework},
  location = {Lisbon, Portugal},
  series = {CAIN '24}
}

In the ever-expanding landscape of Artificial Intelligence (AI), where innovation thrives and new products and services are continuously being delivered, ensuring that AI systems are designed and developed responsibly throughout their entire lifecycle is crucial. To this end, several AI ethics principles and guidelines have been issued to which AI systems should conform. Nevertheless, relying solely on high-level AI ethics principles is far from sufficient to ensure the responsible engineering of AI systems. In this field, AI professionals often navigate by sight. Indeed, while recommendations promoting Trustworthy AI (TAI) exist, they are often high-level statements difficult to translate into concrete implementation strategies. Currently, there is a significant gap between high-level AI ethics principles and low-level concrete practices for AI professionals. To address this challenge, our work presents an experience report where we develop a novel holistic framework for Trustworthy AI — designed to bridge the gap between theory and practice — and report insights from its application in an industrial case study. The framework builds up from the results of a systematic review of the state of the practice as well as a survey and think-aloud interviews with 34 AI practitioners. The framework, unlike most of the ones in literature, is designed to provide actionable guidelines and tools to support different types of stakeholders throughout the entire Software Development Life Cycle (SDLC). Our goal is to empower AI professionals to confidently navigate the ethical dimensions of TAI through practical insights, ensuring that the vast potential of AI is exploited responsibly for the benefit of society as a whole.

@inproceedings{BaldassarreGKRT24,
  author = {Baldassarre, Maria Teresa and Gigante, Domenico and Kalinowski, Marcos and Ragone, Azzurra and Tibid\`{o}, Sara},
  title = {Trustworthy AI in practice: an analysis of practitioners' needs and challenges},
  year = {2024},
  isbn = {9798400717017},
  publisher = {Association for Computing Machinery},
  urlAuthor_version = {http://www.inf.puc-rio.br/~kalinowski/publications/BaldassarreGKRT24.pdf},  
  doi = {10.1145/3661167.3661214},
  abstract = {Recently, there has been growing attention on behalf of both academic and practice communities towards the ability of Artificial Intelligence (AI) systems to operate responsibly and ethically. As a result, a plethora of frameworks and guidelines have appeared to support practitioners in implementing Trustworthy AI applications (TAI). However, little research has been done to investigate whether such frameworks are being used and how. In this work, we study the vision AI practitioners have on TAI principles, how they address them, and what they would like to have – in terms of tools, knowledge, or guidelines – when they attempt to incorporate such principles into the systems they develop. Through a survey and semi-structured interviews, we systematically investigated practitioners’ challenges and needs in developing TAI systems. Based on these practical findings, we highlight recommendations to help AI practitioners develop Trustworthy AI applications.},
  booktitle = {Proceedings of the 28th International Conference on Evaluation and Assessment in Software Engineering},
  pages = {293–302},
  numpages = {10},
  keywords = {Artificial Intelligence, Mixed-method Research, Semi-structured Interview, Software Engineering, Survey, Systematic Investigation, Trustworthy AI},
  location = {Salerno, Italy},
  series = {EASE '24}
}

Recently, there has been growing attention on behalf of both academic and practice communities towards the ability of Artificial Intelligence (AI) systems to operate responsibly and ethically. As a result, a plethora of frameworks and guidelines have appeared to support practitioners in implementing Trustworthy AI applications (TAI). However, little research has been done to investigate whether such frameworks are being used and how. In this work, we study the vision AI practitioners have on TAI principles, how they address them, and what they would like to have – in terms of tools, knowledge, or guidelines – when they attempt to incorporate such principles into the systems they develop. Through a survey and semi-structured interviews, we systematically investigated practitioners’ challenges and needs in developing TAI systems. Based on these practical findings, we highlight recommendations to help AI practitioners develop Trustworthy AI applications.

@inproceedings{BarchilonLKP24,
  author = {Ney Barchilon and Hélio Lopes and Marcos Kalinowski and Jefry Perez},
  title = {Enriquecimento de Dados com Base em Estatísticas de Grafo de Similaridade para Melhorar o Desempenho em Modelos de ML Supervisionados de Classificação},
  abstract = {Esta pesquisa propõe um método para o enriquecimento de conjuntos de dados tabulares utilizando estatísticas de grafo, visando melhorar o desempenho de modelos de ML supervisionados de classificação. O método constrói um grafo a partir da similaridade entre as instâncias do conjunto de dados e extrai características do grafo para enriquecer o conjunto de dados original. Avaliado em 10 conjuntos de dados públicos de diferentes áreas do conhecimento, com 7 modelos de aprendizado de máquina, o método proporcionou um aumento médio de 4,9% na acurácia. Os resultados demonstram a efetividade do método como uma alternativa para melhorar o desempenho de modelos em cenários que conjuntos de dados carecem das características necessárias para as abordagens tradicionais de enriquecimento com a utilização de grafo.},
  booktitle = {Anais do XXXIX Simpósio Brasileiro de Bancos de Dados},
  location = {Florianópolis/SC},
  year = {2024},
  keywords = {Aprendizado de Máquina, Grafo, Similaridade, Grafo por Similaridade, Predição, Enriquecimento de Dados, Características Topológicas Grafo, Redes Complexas},
  issn = {2763-8979},
  pages = {220--233},
  publisher = {SBC},
  doi = {10.5753/sbbd.2024.240196},
  urlAuthor_version = {http://www.inf.puc-rio.br/~kalinowski/publications/BarchilonLKP24.pdf}
}

Esta pesquisa propõe um método para o enriquecimento de conjuntos de dados tabulares utilizando estatísticas de grafo, visando melhorar o desempenho de modelos de ML supervisionados de classificação. O método constrói um grafo a partir da similaridade entre as instâncias do conjunto de dados e extrai características do grafo para enriquecer o conjunto de dados original. Avaliado em 10 conjuntos de dados públicos de diferentes áreas do conhecimento, com 7 modelos de aprendizado de máquina, o método proporcionou um aumento médio de 4,9% na acurácia. Os resultados demonstram a efetividade do método como uma alternativa para melhorar o desempenho de modelos em cenários que conjuntos de dados carecem das características necessárias para as abordagens tradicionais de enriquecimento com a utilização de grafo.

@inproceedings{BusquimALK24,
  author = {Gabriel Busquim and Allysson Araújo and Maria Lima and Marcos Kalinowski},
  title = {Towards Effective Collaboration between Software Engineers and Data Scientists developing Machine Learning-Enabled Systems},
  abstract = {Incorporating Machine Learning (ML) into existing systems is a demand that has grown among several organizations. However, the development of ML-enabled systems encompasses several social and technical challenges, which must be addressed by actors with different fields of expertise working together. This paper has the objective of understanding how to enhance the collaboration between two key actors in building these systems: software engineers and data scientists. We conducted two focus group sessions with experienced data scientists and software engineers working on real-world ML-enabled systems to assess the relevance of different recommendations for specific technical tasks. Our research has found that collaboration between these actors is important for effectively developing ML-enabled systems, especially when defining data access and ML model deployment. Participants provided concrete examples of how recommendations depicted in the literature can benefit collaboration during different tasks. For example, defining clear responsibilities for each team member and creating concise documentation can improve communication and overall performance. Our study contributes to a better understanding of how to foster effective collaboration between software engineers and data scientists creating ML-enabled systems.},
  booktitle = {Anais do XXXVIII Simpósio Brasileiro de Engenharia de Software},
  location = {Curitiba/PR},
  year = {2024},
  keywords = {},
  issn = {0000-0000},
  pages = {24--34},
  publisher = {SBC},
  doi = {10.5753/sbes.2024.3027},
  urlAuthor_version = {http://www.inf.puc-rio.br/~kalinowski/publications/BusquimALK24.pdf}
}

Incorporating Machine Learning (ML) into existing systems is a demand that has grown among several organizations. However, the development of ML-enabled systems encompasses several social and technical challenges, which must be addressed by actors with different fields of expertise working together. This paper has the objective of understanding how to enhance the collaboration between two key actors in building these systems: software engineers and data scientists. We conducted two focus group sessions with experienced data scientists and software engineers working on real-world ML-enabled systems to assess the relevance of different recommendations for specific technical tasks. Our research has found that collaboration between these actors is important for effectively developing ML-enabled systems, especially when defining data access and ML model deployment. Participants provided concrete examples of how recommendations depicted in the literature can benefit collaboration during different tasks. For example, defining clear responsibilities for each team member and creating concise documentation can improve communication and overall performance. Our study contributes to a better understanding of how to foster effective collaboration between software engineers and data scientists creating ML-enabled systems.

@InProceedings{BusquimVLK24,
  author="Busquim, Gabriel
  and Villamizar, Hugo
  and Lima, Maria Julia
  and Kalinowski, Marcos",
  editor="Bludau, Peter
  and Ramler, Rudolf
  and Winkler, Dietmar
  and Bergsmann, Johannes",
  title="On the Interaction Between Software Engineers and Data Scientists When Building Machine Learning-Enabled Systems",
  booktitle= {16th International Conference on Software Quality, Software Quality Days {SWQD} 2024, Vienna, Austria, April 23-25},
  year="2024",
  publisher="Springer Nature Switzerland",
  address="Cham",
  pages="55--75",
  abstract="In recent years, Machine Learning (ML) components have been increasingly integrated into the core systems of organizations. Engineering such systems presents various challenges from both a theoretical and practical perspective. One of the key challenges is the effective interaction between actors with different backgrounds who need to work closely together, such as software engineers and data scientists. This paper presents an exploratory case study that aims to understand the current interaction and collaboration dynamics between these two roles in ML projects. We conducted semi-structured interviews with four practitioners with experience in software engineering and data science of a large ML-enabled system project and analyzed the data using reflexive thematic analysis. Our findings reveal several challenges that can hinder collaboration between software engineers and data scientists, including differences in technical expertise, unclear definitions of each role's duties, and the lack of documents that support the specification of the ML-enabled system. We also indicate potential solutions to address these challenges, such as fostering a collaborative culture, encouraging team communication, and producing concise system documentation. This study contributes to understanding the complex dynamics between software engineers and data scientists in ML projects and provides insights for improving collaboration and communication in this context. We encourage future studies investigating this interaction in other projects.",
  isbn="978-3-031-56281-5",
  doi="10.1007/978-3-031-56281-5_4",
  urlAuthor_version = {http://www.inf.puc-rio.br/~kalinowski/publications/BusquimVLK24.pdf}    
}

In recent years, Machine Learning (ML) components have been increasingly integrated into the core systems of organizations. Engineering such systems presents various challenges from both a theoretical and practical perspective. One of the key challenges is the effective interaction between actors with different backgrounds who need to work closely together, such as software engineers and data scientists. This paper presents an exploratory case study that aims to understand the current interaction and collaboration dynamics between these two roles in ML projects. We conducted semi-structured interviews with four practitioners with experience in software engineering and data science of a large ML-enabled system project and analyzed the data using reflexive thematic analysis. Our findings reveal several challenges that can hinder collaboration between software engineers and data scientists, including differences in technical expertise, unclear definitions of each role's duties, and the lack of documents that support the specification of the ML-enabled system. We also indicate potential solutions to address these challenges, such as fostering a collaborative culture, encouraging team communication, and producing concise system documentation. This study contributes to understanding the complex dynamics between software engineers and data scientists in ML projects and provides insights for improving collaboration and communication in this context. We encourage future studies investigating this interaction in other projects.

@inproceedings{CabralKBVEL24,
  author = {Cabral, Raphael and Kalinowski, Marcos and Baldassarre, Maria Teresa and Villamizar, Hugo and Escovedo, Tatiana and Lopes, H\'{e}lio},
  title = {Investigating the Impact of SOLID Design Principles on Machine Learning Code Understanding},
  note={<font color="red">Best Paper of CAIN 2024 - ACM Distinguished Paper Award!</font>},
  year = {2024},
  isbn = {9798400705915},
  publisher = {Association for Computing Machinery},
  urlAuthor_version = {http://www.inf.puc-rio.br/~kalinowski/publications/CabralKBVEL24.pdf},  
  doi = {10.1145/3644815.3644957},
  abstract = {[Context] Applying design principles has long been acknowledged as beneficial for understanding and maintainability in traditional software projects. These benefits may similarly hold for Machine Learning (ML) projects, which involve iterative experimentation with data, models, and algorithms. However, ML components are often developed by data scientists with diverse educational backgrounds, potentially resulting in code that doesn't adhere to software design best practices. [Goal] In order to better understand this phenomenon, we investigated the impact of the SOLID design principles on ML code understanding. [Method] We conducted a controlled experiment with three independent trials involving 100 data scientists. We restructured real industrial ML code that did not use SOLID principles. Within each trial, one group was presented with the original ML code, while the other was presented with ML code incorporating SOLID principles. Participants of both groups were asked to analyze the code and fill out a questionnaire that included both open-ended and closed-ended questions on their understanding. [Results] The study results provide statistically significant evidence that the adoption of the SOLID design principles can improve code understanding within the realm of ML projects. [Conclusion] We put forward that software engineering design principles should be spread within the data science community and considered for enhancing the maintainability of ML code.},
  booktitle = {Proceedings of the IEEE/ACM 3rd International Conference on AI Engineering - Software Engineering for AI},
  pages = {7–17},
  numpages = {11},
  keywords = {SOLID design principles, machine learning, code understanding},
  location = {Lisbon, Portugal},
  series = {CAIN '24}
}

[Context] Applying design principles has long been acknowledged as beneficial for understanding and maintainability in traditional software projects. These benefits may similarly hold for Machine Learning (ML) projects, which involve iterative experimentation with data, models, and algorithms. However, ML components are often developed by data scientists with diverse educational backgrounds, potentially resulting in code that doesn't adhere to software design best practices. [Goal] In order to better understand this phenomenon, we investigated the impact of the SOLID design principles on ML code understanding. [Method] We conducted a controlled experiment with three independent trials involving 100 data scientists. We restructured real industrial ML code that did not use SOLID principles. Within each trial, one group was presented with the original ML code, while the other was presented with ML code incorporating SOLID principles. Participants of both groups were asked to analyze the code and fill out a questionnaire that included both open-ended and closed-ended questions on their understanding. [Results] The study results provide statistically significant evidence that the adoption of the SOLID design principles can improve code understanding within the realm of ML projects. [Conclusion] We put forward that software engineering design principles should be spread within the data science community and considered for enhancing the maintainability of ML code.

@inproceedings{CabralK24,
  author = {Cabral, Raphael and Kalinowski, Marcos},
  title = {Investigating the Impact of SOLID Design Principles on Machine Learning Code Understanding},
  year = {2024},
  isbn = {9798400717772},
  publisher = {Association for Computing Machinery},
  urlAuthor_version = {http://www.inf.puc-rio.br/~kalinowski/publications/CabralK24.pdf},
  doi = {10.1145/3701625.3701695},
  abstract = {[Context] Applying design principles has long been acknowledged as beneficial for understanding and maintainability in traditional software projects. These benefits may similarly hold for Machine Learning (ML) projects, which involve iterative experimentation with data, models, and algorithms. However, ML components are often developed by data scientists with diverse educational backgrounds, potentially resulting in code that doesn’t adhere to software design best practices. [Goal] To better understand this phenomenon, we investigated the impact of the SOLID design principles on ML code understanding. [Method] We conducted a controlled experiment with three independent trials involving 100 data scientists. We restructured real industrial ML code that did not use SOLID principles. Within each trial, one group was presented with the original ML code, while the other was presented with ML code incorporating SOLID principles. Participants of both groups were asked to analyze the code and fill out a questionnaire that included both open-ended and closed-ended questions on their understanding. [Results] The dissertation results provide statistically significant evidence that adopting the SOLID design principles can improve code understanding within ML projects. [Conclusion] We put forward that software engineering design principles should be spread within the data science community and considered for enhancing the quality and maintainability of ML code.},
  booktitle = {Proceedings of the XXIII Brazilian Symposium on Software Quality},
  pages = {703–705},
  numpages = {3},
  keywords = {SOLID Design Principles, Machine Learning, Code Understanding},
  location = {Salvador, Brazil},
  note = {<font color="red">Summary for the "Third Best Brazilian Software Quality MS Dissertation Award", received at SBQS 2024. Student: Raphael Cabral, Advisor: Marcos Kalinowski.</font>},
  series = {SBQS '24}
}

[Context] Applying design principles has long been acknowledged as beneficial for understanding and maintainability in traditional software projects. These benefits may similarly hold for Machine Learning (ML) projects, which involve iterative experimentation with data, models, and algorithms. However, ML components are often developed by data scientists with diverse educational backgrounds, potentially resulting in code that doesn’t adhere to software design best practices. [Goal] To better understand this phenomenon, we investigated the impact of the SOLID design principles on ML code understanding. [Method] We conducted a controlled experiment with three independent trials involving 100 data scientists. We restructured real industrial ML code that did not use SOLID principles. Within each trial, one group was presented with the original ML code, while the other was presented with ML code incorporating SOLID principles. Participants of both groups were asked to analyze the code and fill out a questionnaire that included both open-ended and closed-ended questions on their understanding. [Results] The dissertation results provide statistically significant evidence that adopting the SOLID design principles can improve code understanding within ML projects. [Conclusion] We put forward that software engineering design principles should be spread within the data science community and considered for enhancing the quality and maintainability of ML code.

@book{EscovedoMK24,
  title = {Introdução à Estatística para Ciência de Dados - Da exploração dos dados à experimentação contínua com exemplos de código em Python e R},
  author = {Escovedo, Tatiana and Marques, Thiago and Kalinowski, Marcos},
  abstract = {Em um mundo cada vez mais data-driven, a capacidade de entender e aplicar conceitos de Estatística na Ciência de Dados se torna cada vez mais essencial para tomadas de decisões estratégicas que podem ter impactos cruciais no negócio. Profissionais da área estão aprofundando seus conhecimentos em estatística aplicada, a fim de desenvolver um conjunto de competências e ferramentas para abordar com eficácia a incerteza subjacente a essas decisões. Por meio da aplicação de métodos estatísticos, os cientistas de dados têm a habilidade de traduzir dados em informações valiosas, transformando o território complexo da incerteza em uma área de atuação clara e informada. Enquanto a Estatística oferece o rigor metodológico e as bases para inferências confiáveis, a Ciência de Dados expande as possibilidades com técnicas computacionais avançadas e modelos complexos, capazes de lidar com o volume, a velocidade e a variedade dos dados modernos. Juntas, essas disciplinas permitem resolver problemas altamente complexos. Neste livro, Tatiana Escovedo, Thiago Marques e Marcos Kalinowski trazem um guia prático e acessível que cobre desde os fundamentos da estatística até suas aplicações em análises exploratórias de dados, modelagem para Machine Learning e experimentação contínua, utilizando Python e R. Com uma abordagem passo a passo, os autores descomplicam a estatística e demonstram sua importância e aplicabilidade na prática da Ciência de Dados. Você terá ainda diversos exemplos práticos para estudar e treinar suas habilidades analíticas para transformar dados em insights relevantes para a tomada de decisões. De fato, muitas vezes é por saber explorar as incertezas que surgem as oportunidades mais significativas, e a Ciência de Dados é o guia confiável para desbravar esse terreno desafiador.},
  pages = {1-205},
  year= {2024},
  publisher = {Casa do C{\'o}digo},
  url = {https://www.casadocodigo.com.br/products/livro-estatistica-datascience}
}

Em um mundo cada vez mais data-driven, a capacidade de entender e aplicar conceitos de Estatística na Ciência de Dados se torna cada vez mais essencial para tomadas de decisões estratégicas que podem ter impactos cruciais no negócio. Profissionais da área estão aprofundando seus conhecimentos em estatística aplicada, a fim de desenvolver um conjunto de competências e ferramentas para abordar com eficácia a incerteza subjacente a essas decisões. Por meio da aplicação de métodos estatísticos, os cientistas de dados têm a habilidade de traduzir dados em informações valiosas, transformando o território complexo da incerteza em uma área de atuação clara e informada. Enquanto a Estatística oferece o rigor metodológico e as bases para inferências confiáveis, a Ciência de Dados expande as possibilidades com técnicas computacionais avançadas e modelos complexos, capazes de lidar com o volume, a velocidade e a variedade dos dados modernos. Juntas, essas disciplinas permitem resolver problemas altamente complexos. Neste livro, Tatiana Escovedo, Thiago Marques e Marcos Kalinowski trazem um guia prático e acessível que cobre desde os fundamentos da estatística até suas aplicações em análises exploratórias de dados, modelagem para Machine Learning e experimentação contínua, utilizando Python e R. Com uma abordagem passo a passo, os autores descomplicam a estatística e demonstram sua importância e aplicabilidade na prática da Ciência de Dados. Você terá ainda diversos exemplos práticos para estudar e treinar suas habilidades analíticas para transformar dados em insights relevantes para a tomada de decisões. De fato, muitas vezes é por saber explorar as incertezas que surgem as oportunidades mais significativas, e a Ciência de Dados é o guia confiável para desbravar esse terreno desafiador.

@inproceedings{GuenesTKBS24,
  author = {Guenes, Paloma and Tomaz, Rafael and Kalinowski, Marcos and Baldassarre, Maria Teresa and Storey, Margaret-Anne},
  title = {Impostor Phenomenon in Software Engineers},
  year = {2024},
  isbn = {9798400704994},
  publisher = {Association for Computing Machinery},
  urlAuthor_version = {http://www.inf.puc-rio.br/~kalinowski/publications/GuenesTKBS24.pdf}, 
  doi = {10.1145/3639475.3640114},
  abstract = {The Impostor Phenomenon (IP) is widely discussed in Science, Technology, Engineering, and Mathematics (STEM) and has been recently evaluated in Computer and Data Science students. There has been no formal research conducted on IP in software engineers in general, even though its consequences may contribute to mental health disorders, such as depression and burnout. This study describes a survey that investigates the extent of impostor feelings in software engineers, considering aspects such as gender, race/ethnicity, and roles. Furthermore, we investigate the influence of IP on their perceived productivity. The survey instrument was designed using a theory-driven approach and included demographic questions, an internationally validated IP scale (CIPS), and questions for measuring perceived productivity based on the SPACE framework constructs. The survey was sent to companies operating in various business sectors. Data analysis used bootstrapping with resampling to calculate confidence intervals and Mann-Whitney statistical significance testing for assessing the hypotheses. We received responses from 624 software engineers distributed across 26 countries. The bootstrapping results reveal that a proportion of 52.7\% of software engineers experience frequent to intense levels of IP and that women suffer at a significantly higher proportion (60.6\%) than men (48.8\%). Regarding race/ethnicity, we observed more frequent impostor feelings in Asian (67.9\%) and Black (65.1\%) than in White (50.0\%) software engineers. We also observed that the presence of IP is less common among individuals who are married and have children. Moreover, the prevalence of IP showed a statistically significant negative effect on the perceived productivity for all SPACE framework constructs. The evidence relating IP to software engineers provides a starting point to help organizations find ways to raise awareness of the problem and improve the emotional skills of software professionals.},
  booktitle = {Proceedings of the 46th International Conference on Software Engineering: Software Engineering in Society},
  pages = {96–106},
  numpages = {11},
  keywords = {impostor phenomenon, imposter syndrome, human aspects, perceived productivity, software engineering},
  location = {Lisbon, Portugal},
  series = {ICSE-SEIS'24}
}

The Impostor Phenomenon (IP) is widely discussed in Science, Technology, Engineering, and Mathematics (STEM) and has been recently evaluated in Computer and Data Science students. There has been no formal research conducted on IP in software engineers in general, even though its consequences may contribute to mental health disorders, such as depression and burnout. This study describes a survey that investigates the extent of impostor feelings in software engineers, considering aspects such as gender, race/ethnicity, and roles. Furthermore, we investigate the influence of IP on their perceived productivity. The survey instrument was designed using a theory-driven approach and included demographic questions, an internationally validated IP scale (CIPS), and questions for measuring perceived productivity based on the SPACE framework constructs. The survey was sent to companies operating in various business sectors. Data analysis used bootstrapping with resampling to calculate confidence intervals and Mann-Whitney statistical significance testing for assessing the hypotheses. We received responses from 624 software engineers distributed across 26 countries. The bootstrapping results reveal that a proportion of 52.7% of software engineers experience frequent to intense levels of IP and that women suffer at a significantly higher proportion (60.6%) than men (48.8%). Regarding race/ethnicity, we observed more frequent impostor feelings in Asian (67.9%) and Black (65.1%) than in White (50.0%) software engineers. We also observed that the presence of IP is less common among individuals who are married and have children. Moreover, the prevalence of IP showed a statistically significant negative effect on the perceived productivity for all SPACE framework constructs. The evidence relating IP to software engineers provides a starting point to help organizations find ways to raise awareness of the problem and improve the emotional skills of software professionals.

@inproceedings{GuenesKB24,
  author = {Paloma Guenes and Marcos Kalinowski and Maria Baldassarre},
  title = {Impostor Phenomenon in Software Engineers},
  booktitle = {Concurso de Teses e Dissertações do XV Congresso Brasileiro de Software: Teoria e Prática},
  note = {<font color="red">Summary for the "Third Best Brazilian Software Engineering MSc Dissertation Award", received at SBES 2024. Student: Paloma Guenes, Advisors: Marcos Kalinowski and Maria Teresa Baldassarre.</font>},  
  location = {Curitiba/PR},
  year = {2024},
  keywords = {},
  issn = {0000-0000},
  pages = {5--6},
  publisher = {SBC},
  doi = {10.5753/cbsoft_estendido.2024.4132},
  urlAuthor_version = {http://www.inf.puc-rio.br/~kalinowski/publications/GuenesKB24.pdf}
}

@inproceedings{JorgeBEK24,
  author = {Jorge, Ezequiel Mule and Barbieri, L\'{u}cio Tales and Escovedo, Tatiana and Kalinowski, Marcos},
  title = {Investigating Predicting Voluntary Resignation Program Participation with Machine Learning},
  year = {2024},
  isbn = {9798400709968},
  publisher = {Association for Computing Machinery},
  doi = {10.1145/3658271.3658326},
  abstract = {Context: The growing challenge in attracting employees to Voluntary Resignation Programs (VRP) lies in the need to balance the company’s cost control with the goal of increasing participation from the target audience. Problem: It is essential to ensure that the process occurs smoothly, reducing tension during the separation and fostering a more cooperative and responsible environment. Companies need to maximize attraction to VRP, minimize costs, and improve resource allocation. Solution: This article aims to construct a Machine Learning (ML) model to predict employee participation VRPs in an organizational context and to identify the key factors that influence employee participation in the program, identifying patterns and trends based on previous programs. IS Theory: This work is associated with the Theory of Computational Learning, which aims to understand the fundamental principles of learning and design better-automated methods. Method: This article constitutes a study of past data, aiming to identify patterns and develop trends related to employee participation through the utilization of ML algorithms. Summary of Results: The investigation into predicting VRP participation using ML revealed compelling correlations. The Bootstrap Aggregating with Logistic Regression model emerged as the most effective, demonstrating high F1-Score and Accuracy. Contributions and Impact in the IS area: The research significantly contributes to the IS field by showcasing ML’s application in predicting VRP participation, enriching our understanding of factors influencing employee decisions and highlights technology-driven solutions in workforce management. Insights from this investigation offer a valuable framework for future research, paving the way for predictive analytics integration in addressing complex HR challenges within the broader IS context.},
  booktitle = {Proceedings of the 20th Brazilian Symposium on Information Systems},
  articleno = {54},
  numpages = {10},
  keywords = {Algorithms, Machine Learning, Models, Prediction, Voluntary Resignation Programs},
  location = {Juiz de Fora, Brazil},
  series = {SBSI '24},
  urlAuthor_version = {http://www.inf.puc-rio.br/~kalinowski/publications/JorgeBEK24.pdf}  
}

Context: The growing challenge in attracting employees to Voluntary Resignation Programs (VRP) lies in the need to balance the company’s cost control with the goal of increasing participation from the target audience. Problem: It is essential to ensure that the process occurs smoothly, reducing tension during the separation and fostering a more cooperative and responsible environment. Companies need to maximize attraction to VRP, minimize costs, and improve resource allocation. Solution: This article aims to construct a Machine Learning (ML) model to predict employee participation VRPs in an organizational context and to identify the key factors that influence employee participation in the program, identifying patterns and trends based on previous programs. IS Theory: This work is associated with the Theory of Computational Learning, which aims to understand the fundamental principles of learning and design better-automated methods. Method: This article constitutes a study of past data, aiming to identify patterns and develop trends related to employee participation through the utilization of ML algorithms. Summary of Results: The investigation into predicting VRP participation using ML revealed compelling correlations. The Bootstrap Aggregating with Logistic Regression model emerged as the most effective, demonstrating high F1-Score and Accuracy. Contributions and Impact in the IS area: The research significantly contributes to the IS field by showcasing ML’s application in predicting VRP participation, enriching our understanding of factors influencing employee decisions and highlights technology-driven solutions in workforce management. Insights from this investigation offer a valuable framework for future research, paving the way for predictive analytics integration in addressing complex HR challenges within the broader IS context.

@incollection{KalinowskiAM24,
  author = {Marcos Kalinowski and Allysson Allex Araújo and Daniel Mendez},
  title = {Teaching Survey Research in Software Engineering},
  editor = {Daniel Mendez and Paris Avgeriou and Marcos Kalinowski and Nauman bin Ali},
  booktitle = {Handbook on Teaching Empirical Software Engineering},
  publisher = {Springer},
  year = {2024},
  pages = {1-27},
  chapter = {Teaching Survey Research in Software Engineering},
  urlAuthor_version = {http://www.inf.puc-rio.br/~kalinowski/publications/KalinowskiAM24.pdf},
  doi = {10.1007/978-3-031-71769-7_18}
}

@inproceedings{KalinowskiELBV24,
  author    = {Marcos Kalinowski and Tatiana Escovedo and Helio Lopes and Simone Barbosa and Adriana Vidal},
  title     = {Implantando uma Metodologia de Ensino a Distância Baseada em MVPs e Integrada com as Necessidades do Mercado},
  booktitle = {XXXII Workshop sobre Educa{\c{c}}{\~a}o em Computa{\c{c}}{\~a}o (CSBC 2024 - WEI 2024), Brasília, Brazil},
  abstract  = {Este artigo relata a experiência da implementação de quatro novos cursos de pós-graduação lato sensu do Departamento de Informática da PUC-Rio. Os cursos foram concebidos com uma metodologia inovadora, incluindo a organização dos cursos em Sprints, nos quais os alunos elaboram Produtos Mínimos Viáveis (MVPs) que avaliam hipóteses de aprendizagem. Os programas foram lançados em fevereiro de 2023 e já contavam com mais de 600 alunos de todo o Brasil ao final da primeira Sprint. A maioria dos alunos conseguiu desenvolver seus MVPs com sucesso, incorporando-os em seus portfólios profissionais e demonstrando seu aprendizado por meio de projetos hands-on próprios. Após um ano, os cursos já tinham recebido mais de 1300 matriculas e formado mais de 500 profissionais.},
  note      = {<font color="red">WEI 2024 Distinguished Paper Award (Men{\c{c}}{\~a}o Honrosa)!</font>},
  pages     = {1-11},
  year      = {2024},
  doi       = {10.5753/wei.2024.3040},
  urlAuthor_version = {http://www.inf.puc-rio.br/~kalinowski/publications/KalinowskiELBV24.pdf}  
}

Este artigo relata a experiência da implementação de quatro novos cursos de pós-graduação lato sensu do Departamento de Informática da PUC-Rio. Os cursos foram concebidos com uma metodologia inovadora, incluindo a organização dos cursos em Sprints, nos quais os alunos elaboram Produtos Mínimos Viáveis (MVPs) que avaliam hipóteses de aprendizagem. Os programas foram lançados em fevereiro de 2023 e já contavam com mais de 600 alunos de todo o Brasil ao final da primeira Sprint. A maioria dos alunos conseguiu desenvolver seus MVPs com sucesso, incorporando-os em seus portfólios profissionais e demonstrando seu aprendizado por meio de projetos hands-on próprios. Após um ano, os cursos já tinham recebido mais de 1300 matriculas e formado mais de 500 profissionais.

@book{MendezAKA24,
  title = {Handbook on Teaching Empirical Software Engineering},
  author = {Daniel Mendez and Paris Avgeriou and Marcos Kalinowski and Nauman bin Ali},
  abstract = {This handbook exploits the profound experience and expertise of well-established scholars in the empirical software engineering community to provide guidance and support in teaching various research methods and fundamental concepts. A particular focus is thus on combining research methods and their epistemological settings and terminology with didactics and pedagogy for the subject. The book covers the most essential contemporary research methods and philosophical and cross-cutting concerns in software engineering research, considering both academic and industrial settings, at the same time providing insights into the effective teaching of concepts and strategies.},
  pages = {1-630},
  year= {2024},
  publisher = {Springer},
  url = {https://link.springer.com/book/9783031717680},
  doi = {10.1007/978-3-031-71769-7}
}

This handbook exploits the profound experience and expertise of well-established scholars in the empirical software engineering community to provide guidance and support in teaching various research methods and fundamental concepts. A particular focus is thus on combining research methods and their epistemological settings and terminology with didactics and pedagogy for the subject. The book covers the most essential contemporary research methods and philosophical and cross-cutting concerns in software engineering research, considering both academic and industrial settings, at the same time providing insights into the effective teaching of concepts and strategies.

@incollection{MendezAKA24b,
  author = {Daniel Mendez and Paris Avgeriou and Marcos Kalinowski and Nauman bin Ali},
  title = {Teaching Empirical Software Engineering: An Editorial Introduction},
  editor = {Daniel Mendez and Paris Avgeriou and Marcos Kalinowski and Nauman bin Ali},
  booktitle = {Handbook on Teaching Empirical Software Engineering},
  publisher = {Springer},
  year = {2024},
  pages = {3-12},
  chapter = {Teaching Empirical Software Engineering: An Editorial Introduction},
  urlAuthor_version = {http://www.inf.puc-rio.br/~kalinowski/publications/MendezAKA24b.pdf},
  doi = {10.1007/978-3-031-71769-7_1}
}

@inproceedings{MoraisEK24,
  author = {Morais, Gustavo and Escovedo, Tatiana and Kalinowski, Marcos},
  title = {Machine Learning Applied in the Construction of a Disability Retirement Entry Table of the General Social Security Regime (RGPS) of Brazil},
  year = {2024},
  isbn = {9798400709968},
  publisher = {Association for Computing Machinery},
  doi = {10.1145/3658271.3658327},
  abstract = {Context: Given the significant expenses incurred by the General Social Security Regime (RGPS) in disability retirement benefit payments, accurately assessing the costs associated with new concessions is crucial for maintaining the financial and actuarial balance of the system. Entry tables for disability retirement are essential instruments for this measurement. Problem: In addition to the scarcity of related studies, the most recent table is based on RGPS concession data from just over twenty years ago, which can impact the precise estimation of costs for new concessions. Solution: This article aims to construct an updated Disability Retirement Entry Table in RGPS, segmented by gender and age, employing Machine Learning models developed using regression algorithms. IS Theory: This article operates under the auspices of Machine Learning Theory, which is related to automating intelligent tasks to identify patterns or mathematical formulations explaining potential data relationships. Method: The research utilizes a quantitative approach, involving the training of Machine Learning models through experiments with regression algorithms and performance evaluation metrics such as RMSE and R2. Summary of Results: The Extra Trees Regressor demonstrated the best performance, achieving results of 0.002039 and 0.917324 for RMSE and R2 metrics, respectively, with training data, and 0.001047 and 0.975334 with out-of-sample data. Contributions and Impact in the IS Area: This article’s primary contribution lies in pioneering the application of Machine Learning to model an entry table for disability retirement. It is expected that the study’s outcomes will stimulate further scientific research in this field and contribute to the construction of other biometric tables, such as general mortality tables, disabled tables, morbidity tables, and more.},
  booktitle = {Proceedings of the 20th Brazilian Symposium on Information Systems},
  articleno = {55},
  numpages = {10},
  keywords = {Algorithms, Disability Entry Tables, General Social Security Regime, Machine Learning},
  location = {Juiz de Fora, Brazil},
  series = {SBSI '24},
  urlAuthor_version = {http://www.inf.puc-rio.br/~kalinowski/publications/MoraisEK24.pdf}
}

Context: Given the significant expenses incurred by the General Social Security Regime (RGPS) in disability retirement benefit payments, accurately assessing the costs associated with new concessions is crucial for maintaining the financial and actuarial balance of the system. Entry tables for disability retirement are essential instruments for this measurement. Problem: In addition to the scarcity of related studies, the most recent table is based on RGPS concession data from just over twenty years ago, which can impact the precise estimation of costs for new concessions. Solution: This article aims to construct an updated Disability Retirement Entry Table in RGPS, segmented by gender and age, employing Machine Learning models developed using regression algorithms. IS Theory: This article operates under the auspices of Machine Learning Theory, which is related to automating intelligent tasks to identify patterns or mathematical formulations explaining potential data relationships. Method: The research utilizes a quantitative approach, involving the training of Machine Learning models through experiments with regression algorithms and performance evaluation metrics such as RMSE and R2. Summary of Results: The Extra Trees Regressor demonstrated the best performance, achieving results of 0.002039 and 0.917324 for RMSE and R2 metrics, respectively, with training data, and 0.001047 and 0.975334 with out-of-sample data. Contributions and Impact in the IS Area: This article’s primary contribution lies in pioneering the application of Machine Learning to model an entry table for disability retirement. It is expected that the study’s outcomes will stimulate further scientific research in this field and contribute to the construction of other biometric tables, such as general mortality tables, disabled tables, morbidity tables, and more.

@inproceedings{NapoleaoEtAl24,
  author = {Napole\~{a}o, Bianca Minetto and Sarkar, Ritika and Hall\'{e}, Sylvain and Petrillo, Fabio and Kalinowski, Marcos},
  title = {Emerging Results on Automated Support for Searching and Selecting Evidence for Systematic Literature Review Updates},
  year = {2024},
  isbn = {9798400705670},
  publisher = {Association for Computing Machinery},
  urlAuthor_version = {http://www.inf.puc-rio.br/~kalinowski/publications/NapoleaoEtAl24.pdf}, 
  doi = {10.1145/3643664.3648202},
  abstract = {Context: The constant growth of primary evidence and Systematic Literature Reviews (SLRs) publications in the Software Engineering (SE) field leads to the need for SLR Updates. However, searching and selecting evidence for SLR updates demands significant effort from SE researchers. Objective: We present emerging results on an automated approach to support searching and selecting studies for SLR updates in SE. Method: We developed an automated tool prototype to perform the snowballing search technique and to support the selection of relevant studies for SLR updates using Machine Learning (ML) algorithms. We evaluated our automation proposition through a small-scale evaluation with a reliable dataset from an SLR replication and its update. Results: Effectively automating snowballing-based search strategies showed feasibility with minor losses, specifically related to papers without Digital Object Identifier (DOI). The ML algorithm giving the highest performance to select studies for SLR updates was Linear Support Vector Machine with approximately 74\% recall and 15\% precision. The use of such algorithms with conservative thresholds to minimize the risk of missing papers can already significantly reduce evidence selection efforts. Conclusion: The preliminary results of our evaluation point in promising directions, indicating the potential of automating snowballing search efforts and of reducing the number of papers to be manually analyzed by about 2.5 times when selecting evidence for updating SLRs in SE.},
  booktitle = {Proceedings of the 1st IEEE/ACM International Workshop on Methodological Issues with Empirical Studies in Software Engineering},
  pages = {34–41},
  numpages = {8},
  keywords = {systematic review update, SLR update, searching for evidence, selecting evidence},
  location = {Lisbon, Portugal},
  series = {WSESE@ICSE '24}
}

Context: The constant growth of primary evidence and Systematic Literature Reviews (SLRs) publications in the Software Engineering (SE) field leads to the need for SLR Updates. However, searching and selecting evidence for SLR updates demands significant effort from SE researchers. Objective: We present emerging results on an automated approach to support searching and selecting studies for SLR updates in SE. Method: We developed an automated tool prototype to perform the snowballing search technique and to support the selection of relevant studies for SLR updates using Machine Learning (ML) algorithms. We evaluated our automation proposition through a small-scale evaluation with a reliable dataset from an SLR replication and its update. Results: Effectively automating snowballing-based search strategies showed feasibility with minor losses, specifically related to papers without Digital Object Identifier (DOI). The ML algorithm giving the highest performance to select studies for SLR updates was Linear Support Vector Machine with approximately 74% recall and 15% precision. The use of such algorithms with conservative thresholds to minimize the risk of missing papers can already significantly reduce evidence selection efforts. Conclusion: The preliminary results of our evaluation point in promising directions, indicating the potential of automating snowballing search efforts and of reducing the number of papers to be manually analyzed by about 2.5 times when selecting evidence for updating SLRs in SE.

@inproceedings{RomaoKBABL24,
  author = {Lucas Romao and Marcos Kalinowski and Clarissa Barbosa and Allysson Araújo and Simone Barbosa and Helio Lopes},
  title = {Agile Minds, Innovative Solutions, and Industry-Academia Collaboration: Lean R&D Meets Problem-Based Learning in Software Engineering Education},
  abstract = {[Context] Software Engineering (SE) education constantly seeks to bridge the gap between academic knowledge and industry demands, with active learning methods like Problem-Based Learning (PBL) gaining prominence. Despite these efforts, recent graduates struggle to align skills with industry needs. Recognizing the relevance of Industry-Academia Collaboration (IAC), Lean R&D has emerged as a successful agile-based research and development approach, emphasizing business and software development synergy. [Goal] This paper aims to extend Lean R&D with PBL principles, evaluating its application in an educational program designed by ExACTa PUC-Rio for Americanas S.A., a large Brazilian retail company. [Method] The educational program engaged 40 part-time students receiving lectures and mentoring while working on real problems, coordinators and mentors, and company stakeholders in industry projects. Empirical evaluation, through a case study approach, utilized struc-tured questionnaires based on the Technology Acceptance Model (TAM). [Results] Stakeholders were satisfied with Lean R&D PBL for problem-solving. Students reported increased knowledge proficiency and perceived working on real problems as contributing the most to their learning. [Conclusion] This research contributes to academia by sharing Lean R&D PBL as an educational IAC approach. For industry, we discuss the implementation of this proposal in an IAC program that promotes workforce skill development and innovative solutions.},  
  booktitle = {Anais do XXXVIII Simpósio Brasileiro de Engenharia de Software},
  location = {Curitiba/PR},
  year = {2024},
  keywords = {},
  issn = {0000-0000},
  pages = {346--356},
  publisher = {SBC},
  doi = {10.5753/sbes.2024.3456},
  urlAuthor_version = {http://www.inf.puc-rio.br/~kalinowski/publications/RomaoKBABL24.pdf}
}

[Context] Software Engineering (SE) education constantly seeks to bridge the gap between academic knowledge and industry demands, with active learning methods like Problem-Based Learning (PBL) gaining prominence. Despite these efforts, recent graduates struggle to align skills with industry needs. Recognizing the relevance of Industry-Academia Collaboration (IAC), Lean R&D has emerged as a successful agile-based research and development approach, emphasizing business and software development synergy. [Goal] This paper aims to extend Lean R&D with PBL principles, evaluating its application in an educational program designed by ExACTa PUC-Rio for Americanas S.A., a large Brazilian retail company. [Method] The educational program engaged 40 part-time students receiving lectures and mentoring while working on real problems, coordinators and mentors, and company stakeholders in industry projects. Empirical evaluation, through a case study approach, utilized struc-tured questionnaires based on the Technology Acceptance Model (TAM). [Results] Stakeholders were satisfied with Lean R&D PBL for problem-solving. Students reported increased knowledge proficiency and perceived working on real problems as contributing the most to their learning. [Conclusion] This research contributes to academia by sharing Lean R&D PBL as an educational IAC approach. For industry, we discuss the implementation of this proposal in an IAC program that promotes workforce skill development and innovative solutions.

@inproceedings{VillamizarK24,
  author = {Hugo Villamizar and Marcos Kalinowski},
  title = {Identifying Concerns When Specifying Machine Learning-Enabled Systems: A Perspective-Based Approach},
  booktitle = {Concurso de Teses e Dissertações do XV Congresso Brasileiro de Software: Teoria e Prática},
  note = {<font color="red">Summary for the "Best Brazilian Software Engineering PhD Thesis Award", received at SBES 2024. Student: Hugo Villamizar, Advisor: Marcos Kalinowski.</font>},   
  location = {Curitiba/PR},
  year = {2024},
  keywords = {},
  issn = {0000-0000},
  pages = {37--38},
  publisher = {SBC},
  doi = {10.5753/cbsoft_estendido.2024.4131},  
  urlAuthor_version = {http://www.inf.puc-rio.br/~kalinowski/publications/VillamizarK24.pdf}
}

@inproceedings{VillamizarK24b,
  author = {Villamizar, Hugo and Kalinowski, Marcos},
  title = {Identifying Concerns When Specifying Machine Learning-Enabled Systems: A Perspective-Based Approach},
  year = {2024},
  isbn = {9798400717772},
  publisher = {Association for Computing Machinery},
  urlAuthor_version = {http://www.inf.puc-rio.br/~kalinowski/publications/VillamizarK24b.pdf},
  doi = {10.1145/3701625.3701696},
  abstract = {Engineering successful machine learning (ML)-enabled systems poses various challenges from both a theoretical and a practical side. Among those challenges are how to effectively address unrealistic expectations of ML capabilities from customers, managers and even other team members, and how to connect business value to engineering and data science activities composed by interdisciplinary teams. In this thesis, we studied the state of the practice and literature of requirements engineering (RE) for ML to propose PerSpecML, a perspective-based approach for specifying ML-enabled systems that helps practitioners identify which attributes, including ML and non-ML components, are important to contribute to the overall system’s quality. The approach involves analyzing 60 concerns related to 28 tasks that practitioners typically face in ML projects, grouping them into five perspectives: system objectives, user experience, infrastructure, model, and data. The conception of  PerSpecML involved a series of validations conducted in different contexts: (i) in academia, (ii) with industry representatives, and (iii) in two real industrial case studies. As a result of the diverse validations and continuous improvements, PerSpecML showed a positive impact to the specification of ML-enabled systems, particularly helping to specify key quality components that would have been otherwise missed without using PerSpecML.},
  booktitle = {Proceedings of the XXIII Brazilian Symposium on Software Quality},
  pages = {673–675},
  numpages = {3},
  keywords = {Requirements Specification, Machine Learning, Software Quality},
  location = {Salvador, Brazil},
  note = {<font color="red">Summary for the "Second Best Brazilian Software Quality PhD Thesis Award", received at SBQS 2024. Student: Hugo Villamizar, Advisor: Marcos Kalinowski.</font>},   
  series = {SBQS '24}
}

Engineering successful machine learning (ML)-enabled systems poses various challenges from both a theoretical and a practical side. Among those challenges are how to effectively address unrealistic expectations of ML capabilities from customers, managers and even other team members, and how to connect business value to engineering and data science activities composed by interdisciplinary teams. In this thesis, we studied the state of the practice and literature of requirements engineering (RE) for ML to propose PerSpecML, a perspective-based approach for specifying ML-enabled systems that helps practitioners identify which attributes, including ML and non-ML components, are important to contribute to the overall system’s quality. The approach involves analyzing 60 concerns related to 28 tasks that practitioners typically face in ML projects, grouping them into five perspectives: system objectives, user experience, infrastructure, model, and data. The conception of PerSpecML involved a series of validations conducted in different contexts: (i) in academia, (ii) with industry representatives, and (iii) in two real industrial case studies. As a result of the diverse validations and continuous improvements, PerSpecML showed a positive impact to the specification of ML-enabled systems, particularly helping to specify key quality components that would have been otherwise missed without using PerSpecML.

@InProceedings{ZimelewiczEtAl24,
  author="Zimelewicz, Eduardo
  and Kalinowski, Marcos
  and Mendez, Daniel
  and Giray, G{\"o}rkem
  and Santos Alves, Antonio Pedro
  and Lavesson, Niklas
  and Azevedo, Kelly
  and Villamizar, Hugo
  and Escovedo, Tatiana
  and Lopes, Helio
  and Biffl, Stefan
  and Musil, Juergen
  and Felderer, Michael
  and Wagner, Stefan
  and Baldassarre, Teresa
  and Gorschek, Tony",
  editor="Bludau, Peter
  and Ramler, Rudolf
  and Winkler, Dietmar
  and Bergsmann, Johannes",
  title="ML-Enabled Systems Model Deployment and Monitoring: Status Quo and Problems",
  booktitle= {16th International Conference on Software Quality, Software Quality Days {SWQD} 2024, Vienna, Austria, April 23-25},
  year="2024",
  publisher="Springer Nature Switzerland",
  address="Cham",
  pages="112--131",
  abstract="[Context] Systems that incorporate Machine Learning (ML) models, often referred to as ML-enabled systems, have become commonplace. However, empirical evidence on how ML-enabled systems are engineered in practice is still limited; this is especially true for activities surrounding ML model dissemination. [Goal] We investigate contemporary industrial practices and problems related to ML model dissemination, focusing on the model deployment and the monitoring ML life cycle phases. [Method] We conducted an international survey to gather practitioner insights on how ML-enabled systems are engineered. We gathered a total of 188 complete responses from 25 countries. We analyze the status quo and problems reported for the model deployment and monitoring phases. We analyzed contemporary practices using bootstrapping with confidence intervals and conducted qualitative analyses on the reported problems applying open and axial coding procedures. [Results] Practitioners perceive the model deployment and monitoring phases as relevant and difficult. With respect to model deployment, models are typically deployed as separate services, with limited adoption of MLOps principles. Reported problems include difficulties in designing the architecture of the infrastructure for production deployment and legacy application integration. Concerning model monitoring, many models in production are not monitored. The main monitored aspects are inputs, outputs, and decisions. Reported problems involve the absence of monitoring practices, the need to create custom monitoring tools, and the selection of suitable metrics. [Conclusion] Our results help provide a better understanding of the adopted practices and problems in practice and support guiding ML deployment and monitoring research in a problem-driven manner.",
  isbn="978-3-031-56281-5",
  urlAuthor_version = {http://www.inf.puc-rio.br/~kalinowski/publications/ZimelewiczEtAl24.pdf}  
}

[Context] Systems that incorporate Machine Learning (ML) models, often referred to as ML-enabled systems, have become commonplace. However, empirical evidence on how ML-enabled systems are engineered in practice is still limited; this is especially true for activities surrounding ML model dissemination. [Goal] We investigate contemporary industrial practices and problems related to ML model dissemination, focusing on the model deployment and the monitoring ML life cycle phases. [Method] We conducted an international survey to gather practitioner insights on how ML-enabled systems are engineered. We gathered a total of 188 complete responses from 25 countries. We analyze the status quo and problems reported for the model deployment and monitoring phases. We analyzed contemporary practices using bootstrapping with confidence intervals and conducted qualitative analyses on the reported problems applying open and axial coding procedures. [Results] Practitioners perceive the model deployment and monitoring phases as relevant and difficult. With respect to model deployment, models are typically deployed as separate services, with limited adoption of MLOps principles. Reported problems include difficulties in designing the architecture of the infrastructure for production deployment and legacy application integration. Concerning model monitoring, many models in production are not monitored. The main monitored aspects are inputs, outputs, and decisions. Reported problems involve the absence of monitoring practices, the need to create custom monitoring tools, and the selection of suitable metrics. [Conclusion] Our results help provide a better understanding of the adopted practices and problems in practice and support guiding ML deployment and monitoring research in a problem-driven manner.

@article{AlmeidaKUF23,
  title = {Negative effects of gamification in education software: Systematic mapping and practitioner perceptions},
  journal = {Information and Software Technology},
  volume = {156},
  pages = {107142},
  year = {2023},
  issn = {0950-5849},
  doi = {10.1016/j.infsof.2022.107142},
  urlAuthor_version = {http://www.inf.puc-rio.br/~kalinowski/publications/AlmeidaKUF23.pdf},
  author = {Cl{\'a}uvin Almeida and Marcos Kalinowski and Anderson Uchôa and Bruno Feijó},
  keywords = {Gamification, Negative effects, Education, Learning, Systematic mapping, Snowballing, Focus group},
  abstract = {Context:
  While most research shows positive effects of gamification, the focus on its adverse effects is considerably smaller and further understanding of these effects is needed.
  Objective:
  To provide a comprehensive overview on research reporting negative effects of game design elements and to provide insights into the awareness of developers on these effects and into how they could be considered in practice.
  Method:
  We conducted a systematic mapping study of the negative effects of game design elements on education/learning systems. We also held a focus group discussion with developers of a gamified software, discussing the mapping study results with regard to their awareness and perceptions on the reported negative effects in practice.
  Results:
  The mapping study revealed 87 papers reporting undesired effects of game design elements. We found that badges, leaderboards, competitions, and points are the game design elements most often reported as causing negative effects. The most cited negative effects were lack of effect, worsened performance, motivational issues, lack of understanding, and irrelevance. The ethical issues of gaming the system and cheating were also often reported. As part of our results, we map the relations between game design elements and the negative effects that they may cause. The focus group revealed that developers were not aware of many of the possible negative effects and that they consider this type of information useful. The discussion revealed their agreement on some of those potential negative effects and also some positive counterparts.
  Conclusions:
  Gamification, when properly applied, can have positive effects on education/learning software. However, gamified software is also prone to generate harmful effects. Revealing and discussing potentially negative effects can help to make more informed decisions considering their trade-off with respect to the expected benefits.}
}

Context: While most research shows positive effects of gamification, the focus on its adverse effects is considerably smaller and further understanding of these effects is needed. Objective: To provide a comprehensive overview on research reporting negative effects of game design elements and to provide insights into the awareness of developers on these effects and into how they could be considered in practice. Method: We conducted a systematic mapping study of the negative effects of game design elements on education/learning systems. We also held a focus group discussion with developers of a gamified software, discussing the mapping study results with regard to their awareness and perceptions on the reported negative effects in practice. Results: The mapping study revealed 87 papers reporting undesired effects of game design elements. We found that badges, leaderboards, competitions, and points are the game design elements most often reported as causing negative effects. The most cited negative effects were lack of effect, worsened performance, motivational issues, lack of understanding, and irrelevance. The ethical issues of gaming the system and cheating were also often reported. As part of our results, we map the relations between game design elements and the negative effects that they may cause. The focus group revealed that developers were not aware of many of the possible negative effects and that they consider this type of information useful. The discussion revealed their agreement on some of those potential negative effects and also some positive counterparts. Conclusions: Gamification, when properly applied, can have positive effects on education/learning software. However, gamified software is also prone to generate harmful effects. Revealing and discussing potentially negative effects can help to make more informed decisions considering their trade-off with respect to the expected benefits.

@article{AlonsoKFBL23,
  title = {A systematic mapping study and practitioner insights on the use of software engineering practices to develop MVPs},
  journal = {Information and Software Technology},
  volume = {156},
  pages = {107144},
  year = {2023},
  issn = {0950-5849},
  doi = {10.1016/j.infsof.2022.107144},
  urlAuthor_version = {http://www.inf.puc-rio.br/~kalinowski/publications/AlonsoKFBL23.pdf},
  author = {Silvio Alonso and Marcos Kalinowski and Bruna Ferreira and Simone D.J. Barbosa and H{\'e}lio Lopes},
  keywords = {MVP, Minimum viable product, Software engineering, Systematic mapping, Focus group},
  abstract = {Background:
  Many startup environments and even traditional software companies have embraced the use of MVPs (Minimum Viable Products) to allow quickly experimenting solution options. The MVP concept has influenced the way in which development teams apply Software Engineering (SE) practices. However, the overall understanding of this influence of MVPs on SE practices is still poor.
  Objective:
  Our goal is to characterize the publication landscape on practices that have been used in the context of software MVPs and to gather practitioner insights on the identified practices.
  Method:
  We conducted a systematic mapping study using a hybrid search strategy that consists of a database search and parallel forward and backward snowballing. Thereafter, we discussed the mapping study results in two focus groups sessions involving twelve industry practitioners that extensively use MVPs in their projects to capture their perceptions on the findings of the mapping study.
  Results:
  We identified 33 papers published between 2013 and 2020. We observed some trends related to MVP ideation (or MVP conception) and evaluation practices. For instance, regarding ideation, we found six different approaches (e.g., Design Thinking, Lean Inception) and mainly informal end-user involvement practices (e.g., workshops, interviews). Regarding evaluation, there is an emphasis on end-user validations based on practices such as usability tests, A/B testing, and usage data analysis. However, there is still limited research related to MVP technical feasibility assessment and effort estimation. Practitioners of the focus group sessions reinforced the confidence in our results regarding ideation and evaluation practices, being aware of most of the identified practices. They also reported how they deal with the technical feasibility assessments (involving developers during the ideation and conducting informal experiments) and effort estimation in practice (based on expert opinion and using practices common to agile methodologies, such as Planning Poker).
  Conclusion:
  Our analysis suggests that there are opportunities for solution proposals and evaluation studies to address literature gaps concerning technical feasibility assessment and effort estimation. Overall, more effort needs to be invested into empirically evaluating the existing MVP-related practices.}
}

Background: Many startup environments and even traditional software companies have embraced the use of MVPs (Minimum Viable Products) to allow quickly experimenting solution options. The MVP concept has influenced the way in which development teams apply Software Engineering (SE) practices. However, the overall understanding of this influence of MVPs on SE practices is still poor. Objective: Our goal is to characterize the publication landscape on practices that have been used in the context of software MVPs and to gather practitioner insights on the identified practices. Method: We conducted a systematic mapping study using a hybrid search strategy that consists of a database search and parallel forward and backward snowballing. Thereafter, we discussed the mapping study results in two focus groups sessions involving twelve industry practitioners that extensively use MVPs in their projects to capture their perceptions on the findings of the mapping study. Results: We identified 33 papers published between 2013 and 2020. We observed some trends related to MVP ideation (or MVP conception) and evaluation practices. For instance, regarding ideation, we found six different approaches (e.g., Design Thinking, Lean Inception) and mainly informal end-user involvement practices (e.g., workshops, interviews). Regarding evaluation, there is an emphasis on end-user validations based on practices such as usability tests, A/B testing, and usage data analysis. However, there is still limited research related to MVP technical feasibility assessment and effort estimation. Practitioners of the focus group sessions reinforced the confidence in our results regarding ideation and evaluation practices, being aware of most of the identified practices. They also reported how they deal with the technical feasibility assessments (involving developers during the ideation and conducting informal experiments) and effort estimation in practice (based on expert opinion and using practices common to agile methodologies, such as Planning Poker). Conclusion: Our analysis suggests that there are opportunities for solution proposals and evaluation studies to address literature gaps concerning technical feasibility assessment and effort estimation. Overall, more effort needs to be invested into empirically evaluating the existing MVP-related practices.

@article{FelipeKGN23,
  title = {Psychometric instruments in software engineering research on personality: Status quo after fifty years},
  journal = {Journal of Systems and Software},
  volume = {203},
  pages = {111740},
  year = {2023},
  issn = {0164-1212},
  doi = {https://doi.org/10.1016/j.jss.2023.111740},
  urlAuthor_version = {http://www.inf.puc-rio.br/~kalinowski/publications/FelipeKGN23.pdf},
  author = {Danilo Almeida Felipe and Marcos Kalinowski and Daniel Graziotin and Jean Carlos Natividade},
  keywords = {Behavioral software engineering, Personality, Mapping study},
  abstract = {Context:
  Although software development is a human activity, Software Engineering (SE) research has focused mostly on processes and tools, making human factors underrepresented. This kind of research may be improved using knowledge from human-focused disciplines. An example of missed opportunities is how SE employs psychometric instruments.
  Objective:
  Provide an overview of psychometric instruments in SE research regarding personality and provide recommendations for adopting them.
  Method:
  We conducted a systematic mapping to build an overview of instruments used within SE for assessing personality and reviewed their use from a multidisciplinary perspective of SE and social science.
  Results:
  We contribute with a secondary study covering fifty years of research (1970 to 2020). One of the most adopted instruments (MBTI) faces criticism within social sciences, and we identified discrepancies between its application and existing recommendations. We emphasize that several instruments refer to the Five-Factor Model, which despite its relevance in social sciences, has no specific advice for its application within SE. We discuss general advice for its proper application.
  Conclusion:
  The findings show that the adoption of psychometric instruments regarding personality in SE needs to be improved, ideally with the support of social science researchers. We believe that the review presented in this study can help to understand limitations and to evolve in this direction.}
}

Context: Although software development is a human activity, Software Engineering (SE) research has focused mostly on processes and tools, making human factors underrepresented. This kind of research may be improved using knowledge from human-focused disciplines. An example of missed opportunities is how SE employs psychometric instruments. Objective: Provide an overview of psychometric instruments in SE research regarding personality and provide recommendations for adopting them. Method: We conducted a systematic mapping to build an overview of instruments used within SE for assessing personality and reviewed their use from a multidisciplinary perspective of SE and social science. Results: We contribute with a secondary study covering fifty years of research (1970 to 2020). One of the most adopted instruments (MBTI) faces criticism within social sciences, and we identified discrepancies between its application and existing recommendations. We emphasize that several instruments refer to the Five-Factor Model, which despite its relevance in social sciences, has no specific advice for its application within SE. We discuss general advice for its proper application. Conclusion: The findings show that the adoption of psychometric instruments regarding personality in SE needs to be improved, ideally with the support of social science researchers. We believe that the review presented in this study can help to understand limitations and to evolve in this direction.

@article{FerreiraMKLB23,
  title = {Lessons learned to improve the UX practices in agile projects involving data science and process automation},
  journal = {Information and Software Technology},
  volume = {155},
  pages = {107106},
  year = {2023},
  issn = {0950-5849},
  doi = {10.1016/j.infsof.2022.107106},
  urlAuthor_version = {http://www.inf.puc-rio.br/~kalinowski/publications/FerreiraMKLB23.pdf},
  author = {Bruna Ferreira and Silvio Marques and Marcos Kalinowski and H{\'e}lio Lopes and Simone D.J. Barbosa},
  keywords = {Agile, User-centered design, Lean inception, User involvement, User participation, User experience, Data science},
  abstract = {Context:
  User-Centered Design (UCD) and Agile methodologies focus on human issues. Nevertheless, agile methodologies focus on contact with contracting customers and generating value for them. Usually, the communication between end users (they use the software and have low decision power) and the agile team is mediated by customers (they have high decision power but do not use the software). However, they do not know the actual problems that end users (may) face in their routine, and they may not be directly affected by software shortcomings. In this context, UX issues are typically identified only after the implementation, during user testing and validation.
  Objective:
  Aiming to improve the understanding and definition of the problem in agile projects, this research investigates the practices and difficulties experienced by agile teams during the development of data science and process automation projects. Also, we analyze the benefits and the teams’ perceptions regarding user participation in these projects.
  Method:
  We collected data from four agile teams, in the context of an academia and industry collaboration focusing on delivering data science and process automation solutions. Therefore, we applied a carefully designed questionnaire answered by developers, scrum masters, and UX designers. In total, 18 subjects answered the questionnaire.
  Results:
  From the results, we identify practices used by the teams to define and understand the problem and to represent the solution. The practices most often used are prototypes and meetings with stakeholders. Another practice that helped the team to understand the problem was using Lean Inception (LI) ideation workshops. Also, our results present some specific issues regarding data science projects.
  Conclusion:
  We observed that end-user participation can be critical to understanding and defining the problem. They help to define elements of the domain and barriers in the implementation. We identified a need for approaches that facilitate user-team communication in data science projects to understand the data and its value to the users’ routine. We also identified insights about the need of more detailed requirements representations to support the development of data science solutions.}
}

Context: User-Centered Design (UCD) and Agile methodologies focus on human issues. Nevertheless, agile methodologies focus on contact with contracting customers and generating value for them. Usually, the communication between end users (they use the software and have low decision power) and the agile team is mediated by customers (they have high decision power but do not use the software). However, they do not know the actual problems that end users (may) face in their routine, and they may not be directly affected by software shortcomings. In this context, UX issues are typically identified only after the implementation, during user testing and validation. Objective: Aiming to improve the understanding and definition of the problem in agile projects, this research investigates the practices and difficulties experienced by agile teams during the development of data science and process automation projects. Also, we analyze the benefits and the teams’ perceptions regarding user participation in these projects. Method: We collected data from four agile teams, in the context of an academia and industry collaboration focusing on delivering data science and process automation solutions. Therefore, we applied a carefully designed questionnaire answered by developers, scrum masters, and UX designers. In total, 18 subjects answered the questionnaire. Results: From the results, we identify practices used by the teams to define and understand the problem and to represent the solution. The practices most often used are prototypes and meetings with stakeholders. Another practice that helped the team to understand the problem was using Lean Inception (LI) ideation workshops. Also, our results present some specific issues regarding data science projects. Conclusion: We observed that end-user participation can be critical to understanding and defining the problem. They help to define elements of the domain and barriers in the implementation. We identified a need for approaches that facilitate user-team communication in data science projects to understand the data and its value to the users’ routine. We also identified insights about the need of more detailed requirements representations to support the development of data science solutions.

@article{MattosEtAl23,
  title = {A framework for enhancing industrial soft sensor learning models},
  journal = {Digital Chemical Engineering},
  volume = {8},
  pages = {100112},
  year = {2023},
  issn = {2772-5081},
  doi = {10.1016/j.dche.2023.100112},
  urlAuthor_version = {http://www.inf.puc-rio.br/~kalinowski/publications/MattosEtAl23.pdf},
  author = {Jo{\~a}o Guilherme Mattos and Patrick Nigri Happ and William Fernandes and Helio Côrtes Vieira Lopes and Simone D J Barbosa and Marcos Kalinowski and Luisa Silveira Rosa and Cassia Novello and Leonardo Dorigo Ribeiro and Patricia Rodrigues Ventura and Marcelo Cardoso Marques and Renato Neves Pitta and Valmir Jose Camolesi and Livia Pereira Lemos Costa and Bruno Itagyba Paravidino and Cristiane Salgado Pereira},
  keywords = {Soft sensors, Data augmentation, Data enhancement, Supervised learning, Semi-supervised learning},
  abstract = {Refinery industrial processes are very complex with nonlinear dynamics resulting from varying feedstock characteristics and also from changes in product prioritization. Along these processes, there are key properties of intermediate compounds that must be monitored and controlled since they directly affect the quality of the end products commercialized by these manufacturers. However, most of these properties can only be measured through time-consuming and expensive laboratory analysis, which is impossible to obtain in high frequencies, as required to properly monitor them. In this sense, developing soft sensors is the most common way to obtain high-frequency estimations for these measurements, helping advanced control systems to establish the correct setpoints for temperatures, pressures, and other sensors along the refining process, controlling the quality of end products. Since the amount of labeled data is scarce, most academic research has focused on employing semi- supervised learning strategies to develop machine learning (ML) models as soft sensors. Our research, on the other hand, goes in another direction. We aim to elaborate a framework that leverages the knowledge of domain experts and employs data augmentation techniques to build an enhanced fully labeled dataset that could be fed to any supervised ML algorithm to generate a quality soft sensor. We applied our framework together with Automated ML to train a model capable of predicting a specific key property associated with the production of Naphtha compounds in a refinery: the ASTM 95% distillation temperature of the Heavy Naphtha. Although our framework is model agnostic, we opted by using Automated ML for the optimization strategy, since it applies a diverse set of models to the dataset, reducing the bias of utilizing a single optimization algorithm. We evaluated the proposed framework on a case study carried out in an industrial refinery in Brazil, where the previous model in production for estimating the ASTM 95% distillation temperature of the Heavy Naphtha was based entirely on the physicochemical knowledge of the process. By adopting our framework with Automated ML, we were capable of improving the R2 score by 120%. The resulting ML model is currently operating in real-time inside the refinery, leading to significant economic gains.}
}

Refinery industrial processes are very complex with nonlinear dynamics resulting from varying feedstock characteristics and also from changes in product prioritization. Along these processes, there are key properties of intermediate compounds that must be monitored and controlled since they directly affect the quality of the end products commercialized by these manufacturers. However, most of these properties can only be measured through time-consuming and expensive laboratory analysis, which is impossible to obtain in high frequencies, as required to properly monitor them. In this sense, developing soft sensors is the most common way to obtain high-frequency estimations for these measurements, helping advanced control systems to establish the correct setpoints for temperatures, pressures, and other sensors along the refining process, controlling the quality of end products. Since the amount of labeled data is scarce, most academic research has focused on employing semi- supervised learning strategies to develop machine learning (ML) models as soft sensors. Our research, on the other hand, goes in another direction. We aim to elaborate a framework that leverages the knowledge of domain experts and employs data augmentation techniques to build an enhanced fully labeled dataset that could be fed to any supervised ML algorithm to generate a quality soft sensor. We applied our framework together with Automated ML to train a model capable of predicting a specific key property associated with the production of Naphtha compounds in a refinery: the ASTM 95% distillation temperature of the Heavy Naphtha. Although our framework is model agnostic, we opted by using Automated ML for the optimization strategy, since it applies a diverse set of models to the dataset, reducing the bias of utilizing a single optimization algorithm. We evaluated the proposed framework on a case study carried out in an industrial refinery in Brazil, where the previous model in production for estimating the ASTM 95% distillation temperature of the Heavy Naphtha was based entirely on the physicochemical knowledge of the process. By adopting our framework with Automated ML, we were capable of improving the R2 score by 120%. The resulting ML model is currently operating in real-time inside the refinery, leading to significant economic gains.

@inproceedings{AlvesEtAl23,
  author = {Antonio Pedro Santos Alves and Marcos Kalinowski and G\"orkem Giray and Daniel Mendez and Niklas Lavesson and Kelly Azevedo and Hugo Villamizar and Tatiana Escovedo and Helio Lopes and Stefan Biffl and J\"urgen Musil and Michael Felderer and Stefan Wagner and Teresa Baldassarre and Tony Gorschek},
  title = {Status Quo and Problems of Requirements Engineering for Machine Learning: Results from an International Survey},
  abstract = {Systems that use Machine Learning (ML) have become commonplace for companies that want to improve their products and processes. Literature suggests that Requirements Engineering (RE) can help address many problems when engineering ML-enabled systems. However, the state of empirical evidence on how RE is applied in practice in the context of ML-enabled systems is mainly dominated by isolated case studies with limited generalizability. We conducted an international survey to gather practitioner insights into the status quo and problems of RE in ML-enabled systems. We gathered 188 complete responses from 25 countries. We conducted quantitative statistical analyses on contemporary practices using bootstrapping with confidence intervals and qualitative analyses on the reported problems involving open and axial coding procedures. We found significant differences in RE practices within ML projects. For instance, (i) RE-related activities are mostly conducted by project leaders and data scientists, (ii) the prevalent requirements documentation format concerns interactive Notebooks, (iii) the main focus of non-functional requirements includes data quality, model reliability, and model explainability, and (iv) main challenges include managing customer expectations and aligning requirements with data. The qualitative analyses revealed that practitioners face problems related to lack of business domain understanding, unclear goals and requirements, low customer engagement, and communication issues. These results help to provide a better understanding of the adopted practices and of which problems exist in practical environments. We put forward the need to adapt further and disseminate RE-related practices for engineering ML-enabled systems.},
  booktitle = {Product-Focused Software Process Improvement - 24th International Conference, {PROFES} 2023, Dornbirn, Austria, December 10-13, 2023. Proceedings},
  note = {<font color="red">Best Paper Candidate, invited for journal extension.</font>},
  year = {2023},
  pages = {1-16},
  urlAuthor_version = {http://www.inf.puc-rio.br/~kalinowski/publications/AlvesEtAl23.pdf},
  doi = {10.1007/978-3-031-49266-2_11}
} 

Systems that use Machine Learning (ML) have become commonplace for companies that want to improve their products and processes. Literature suggests that Requirements Engineering (RE) can help address many problems when engineering ML-enabled systems. However, the state of empirical evidence on how RE is applied in practice in the context of ML-enabled systems is mainly dominated by isolated case studies with limited generalizability. We conducted an international survey to gather practitioner insights into the status quo and problems of RE in ML-enabled systems. We gathered 188 complete responses from 25 countries. We conducted quantitative statistical analyses on contemporary practices using bootstrapping with confidence intervals and qualitative analyses on the reported problems involving open and axial coding procedures. We found significant differences in RE practices within ML projects. For instance, (i) RE-related activities are mostly conducted by project leaders and data scientists, (ii) the prevalent requirements documentation format concerns interactive Notebooks, (iii) the main focus of non-functional requirements includes data quality, model reliability, and model explainability, and (iv) main challenges include managing customer expectations and aligning requirements with data. The qualitative analyses revealed that practitioners face problems related to lack of business domain understanding, unclear goals and requirements, low customer engagement, and communication issues. These results help to provide a better understanding of the adopted practices and of which problems exist in practical environments. We put forward the need to adapt further and disseminate RE-related practices for engineering ML-enabled systems.

@inproceedings{CalefatoQLK23,
  author = {Fabio Calefato and Luigi Quaranta and Filippo Lanubile and Marcos Kalinowski},
  title = {Assessing the Use of AutoML for Data-Driven Software Engineering},
  abstract = {Background. Due to the widespread adoption of Artificial Intelligence (AI) and Machine Learning (ML) for building software applications, companies are struggling to recruit employees with a deep understanding of such technologies. In this scenario, AutoML is soaring as a promising solution to fill the AI/ML skills gap since it promises to automate the building of end-to-end AI/ML pipelines that would normally be engineered by specialized team members. Aims. Despite the growing interest and high expectations, there is a dearth of information about the extent to which AutoML is currently adopted by teams developing AI/ML-enabled systems and how it is perceived by practitioners and researchers. Method. To fill these gaps, in this paper, we present a mixed-method study comprising a benchmark of 12 end-to-end AutoML tools on two SE datasets and a user survey with follow-up interviews to further our understanding of AutoML adoption and perception. Results. We found that AutoML solutions can generate models that outperform those trained and optimized by researchers to perform classification tasks in the SE domain. Also, our findings show that the currently available AutoML solutions do not live up to their names as they do not equally support automation across the stages of the ML development workflow and for all the team members. Conclusions. We derive insights to inform the SE research community on how AutoML can facilitate their activities and tool builders on how to design the next generation of AutoML technologies.},
  booktitle = {Proceedings of the 17th {ACM/IEEE} International Symposium on Empirical Software Engineering and Measurement, {ESEM} 2023, New Orleans, USA, October 26-27},
  year = {2023},
  pages = {1-11},
  urlAuthor_version = {http://www.inf.puc-rio.br/~kalinowski/publications/CalefatoQLK23.pdf},
  doi = {}
} 

Background. Due to the widespread adoption of Artificial Intelligence (AI) and Machine Learning (ML) for building software applications, companies are struggling to recruit employees with a deep understanding of such technologies. In this scenario, AutoML is soaring as a promising solution to fill the AI/ML skills gap since it promises to automate the building of end-to-end AI/ML pipelines that would normally be engineered by specialized team members. Aims. Despite the growing interest and high expectations, there is a dearth of information about the extent to which AutoML is currently adopted by teams developing AI/ML-enabled systems and how it is perceived by practitioners and researchers. Method. To fill these gaps, in this paper, we present a mixed-method study comprising a benchmark of 12 end-to-end AutoML tools on two SE datasets and a user survey with follow-up interviews to further our understanding of AutoML adoption and perception. Results. We found that AutoML solutions can generate models that outperform those trained and optimized by researchers to perform classification tasks in the SE domain. Also, our findings show that the currently available AutoML solutions do not live up to their names as they do not equally support automation across the stages of the ML development workflow and for all the team members. Conclusions. We derive insights to inform the SE research community on how AutoML can facilitate their activities and tool builders on how to design the next generation of AutoML technologies.

@inproceedings{KalinowskiEPVPPG23,
  author = {Kalinowski, Marcos and Escovedo, Tatiana and Pina, Fernanda and Vidal, Adriana and Pereira, Ariane and Ponsirenas, Ricardo and Garibaldi da Rocha, Daiana},
  title = {Training the Professionals that Industry Needs: The Digital Software Engineering Education Program at PUC-Rio},
  year = {2023},
  isbn = {},
  publisher = {Association for Computing Machinery},
  urlAuthor_version = {http://www.inf.puc-rio.br/~kalinowski/publications/KalinowskiEPVPPG23.pdf},
  doi = {},
  abstract = {Software is strategic for Brazil's development, but the lack of a larger qualified workforce limits the country's productive capacity. This paper reports the experience of deploying a large-scale distance learning education program to meet the practical needs of the software industry. We applied design thinking involving software engineering and continued education experts of PUC-Rio and an external consultancy to design a methodology that we consider ideal, including learning objects hitherto not used in continued education in Brazil. The key elements of the conceived methodology include (i) the organization of the course in Sprints that involve the elaboration in practice, with synchronous and asynchronous assistance, of Minimum Viable Products (MVPs) that evaluate learning hypotheses associated with the disciplines; (ii) premium materials with innovative learning objects co-created by professors that have theoretical depth and project experience in their subject areas and professional instruction designers; and (iii) industry integration through React videos and synchronous meetings with reference professionals. We conducted a pilot study implementing the methodology for one discipline and gathering feedback from stakeholders. After validating and adjusting the methodology, the professional education program was launched in February 2023 and by July of that same year it already had more than 300 students from all over Brazil. Most students successfully developed their MVPs, integrating them into their professional portfolio and demonstrating the acquired expertise. The students' satisfaction with the methodology is widely and easily evidenced. The digital Software Engineering education program of PUC-Rio is now a reality and is helping prepare the future workforce for the software area; we share the experience and lessons learned with the community.},
  booktitle = {Proceedings of the XXXVII Brazilian Symposium on Software Engineering},
  pages = {1–10},
  numpages = {10},
  keywords = {Professional education, Distance learning, Software engineering, Industry integration, Minimum viable product},
  location = {Campo Grande, Brazil},
  series = {SBES '23}
}

Software is strategic for Brazil's development, but the lack of a larger qualified workforce limits the country's productive capacity. This paper reports the experience of deploying a large-scale distance learning education program to meet the practical needs of the software industry. We applied design thinking involving software engineering and continued education experts of PUC-Rio and an external consultancy to design a methodology that we consider ideal, including learning objects hitherto not used in continued education in Brazil. The key elements of the conceived methodology include (i) the organization of the course in Sprints that involve the elaboration in practice, with synchronous and asynchronous assistance, of Minimum Viable Products (MVPs) that evaluate learning hypotheses associated with the disciplines; (ii) premium materials with innovative learning objects co-created by professors that have theoretical depth and project experience in their subject areas and professional instruction designers; and (iii) industry integration through React videos and synchronous meetings with reference professionals. We conducted a pilot study implementing the methodology for one discipline and gathering feedback from stakeholders. After validating and adjusting the methodology, the professional education program was launched in February 2023 and by July of that same year it already had more than 300 students from all over Brazil. Most students successfully developed their MVPs, integrating them into their professional portfolio and demonstrating the acquired expertise. The students' satisfaction with the methodology is widely and easily evidenced. The digital Software Engineering education program of PUC-Rio is now a reality and is helping prepare the future workforce for the software area; we share the experience and lessons learned with the community.

@inproceedings{AbbehusenROEK23,
  author = {Abbehusen, Marcelo and Rodrigues, Rodolfo and Oliveira, Lucas and Escovedo, Tatiana and Kalinowski, Marcos},
  title = {Optimizing Wireline Formation Testing in Oil Wells: A Data Science Approach},
  year = {2023},
  isbn = {9798400707599},
  publisher = {Association for Computing Machinery},
  urlAuthor_version = {http://www.inf.puc-rio.br/~kalinowski/publications/AbbehusenROEK23.pdf},
  doi = {10.1145/3592813.3592895},
  abstract = {Context: Wireline Formation Testers are common operations in well testing and evaluation, as they provide key information for exploration and production activities, such as reservoir pressure and permeability. The operation is conducted by following a line graph, where the X and Y axes are time and pressure, respectively. Problem: The decision on whether to end or not the operation is based on this graph. Unfortunately, there is no consensus on the ideal method to conduct these operations, and that is precisely the objective of this work. Solution: Using mathematical modeling, optimization and data science techniques, this work aims to increase the efficiency of wireline formation testing, defining objective metrics for operations conductions, in order to reduce time and guarantee data quality.IS Theory: This work is associated with the Theory of Computational Learning, which aims to understand the fundamental principles of learning and design better-automated methods. Method: The research has a prescriptive character, following machine learning model building best-practices and using a quantitative approach in analyzing results. Summary of Results: The results obtained show that there is a clear potential for reducing operating time and, therefore, costs, if the proposed methodology is used in routine operations. Contributions and Impact in the IS area: This article shows that, through mathematical modeling and the application of data science techniques, it is possible to significantly reduce the time of Wireline Formation Test operations, without any relevant loss of information, which can be a significant gain for oil and gas companies.},
  booktitle = {Proceedings of the XIX Brazilian Symposium on Information Systems},
  pages = {112–119},
  numpages = {8},
  keywords = {wireline, Formation test, formation pressure, machine learning, optimization, well logging},
  location = {Macei\'{o}, Brazil},
  series = {SBSI '23}
}

Context: Wireline Formation Testers are common operations in well testing and evaluation, as they provide key information for exploration and production activities, such as reservoir pressure and permeability. The operation is conducted by following a line graph, where the X and Y axes are time and pressure, respectively. Problem: The decision on whether to end or not the operation is based on this graph. Unfortunately, there is no consensus on the ideal method to conduct these operations, and that is precisely the objective of this work. Solution: Using mathematical modeling, optimization and data science techniques, this work aims to increase the efficiency of wireline formation testing, defining objective metrics for operations conductions, in order to reduce time and guarantee data quality.IS Theory: This work is associated with the Theory of Computational Learning, which aims to understand the fundamental principles of learning and design better-automated methods. Method: The research has a prescriptive character, following machine learning model building best-practices and using a quantitative approach in analyzing results. Summary of Results: The results obtained show that there is a clear potential for reducing operating time and, therefore, costs, if the proposed methodology is used in routine operations. Contributions and Impact in the IS area: This article shows that, through mathematical modeling and the application of data science techniques, it is possible to significantly reduce the time of Wireline Formation Test operations, without any relevant loss of information, which can be a significant gain for oil and gas companies.

@inproceedings{AmaralATEK23,
  author = {Amaral, Andr\'{e} and Ara\'{u}jo, Andr\'{e} and Tomazela, Bruno and Escovedo, Tatiana and Kalinowski, Marcos},
  title = {Machine Learning Applied to the Classification of Technical Inspection Recommendations Regarding the Trend to Increase Criticality},
  year = {2023},
  isbn = {9798400707599},
  publisher = {Association for Computing Machinery},
  urlAuthor_version = {http://www.inf.puc-rio.br/~kalinowski/publications/AmaralATEK23.pdf},
  doi = {10.1145/3592813.3592884},
  abstract = {Context: The problem of prioritizing maintenance activities has become a subject of great interest to the industry in a highly competitive scenario where profitability, productivity, and safety are sought in highly automated plant operations. Problem: The operational unit object of this study faces a high demand for maintenance services, indicated by the technical inspection recommendations (TIR) and the risks arising from not meeting them in a timely manner, with the consequent operational and financial losses. Solution: This article proposes a tool to support the prioritization of equipment maintenance activities through a machine learning model that classifies TIRs according to their criticality, thus providing a second opinion for the classification made by the inspector and, therefore, indicating those that have a greater tendency to escalate in criticality. IS Theory: This work is associated with the Theory of the knowledge-based company, assisting in decision-making and efficiency in the application of resources. Method: To construct the Machine Learning model, algorithms and ensembles used in classification problems and natural language processing methods were used, considering that one of the most relevant attributes of the dataset is a free textual field. Summary of Results: After carrying out several experiments, we arrived at the best performance model, with a recall of 87.08 percent and an accuracy of 95.19 percent, values ​​understood as promising for the implementation of the tool in production. Contributions and Impact in the IS area: The main contribution was to identify the feasibility of using machine learning models for the classification of technical inspection recommendations regarding the trend to increase criticality.},
  booktitle = {Proceedings of the XIX Brazilian Symposium on Information Systems},
  pages = {17–27},
  numpages = {11},
  keywords = {Service prioritization, Machine learning, Classification models, Natural Language Processing, Industrial maintenance},
  location = {Macei\'{o}, Brazil},
  series = {SBSI '23}
}

Context: The problem of prioritizing maintenance activities has become a subject of great interest to the industry in a highly competitive scenario where profitability, productivity, and safety are sought in highly automated plant operations. Problem: The operational unit object of this study faces a high demand for maintenance services, indicated by the technical inspection recommendations (TIR) and the risks arising from not meeting them in a timely manner, with the consequent operational and financial losses. Solution: This article proposes a tool to support the prioritization of equipment maintenance activities through a machine learning model that classifies TIRs according to their criticality, thus providing a second opinion for the classification made by the inspector and, therefore, indicating those that have a greater tendency to escalate in criticality. IS Theory: This work is associated with the Theory of the knowledge-based company, assisting in decision-making and efficiency in the application of resources. Method: To construct the Machine Learning model, algorithms and ensembles used in classification problems and natural language processing methods were used, considering that one of the most relevant attributes of the dataset is a free textual field. Summary of Results: After carrying out several experiments, we arrived at the best performance model, with a recall of 87.08 percent and an accuracy of 95.19 percent, values ​​understood as promising for the implementation of the tool in production. Contributions and Impact in the IS area: The main contribution was to identify the feasibility of using machine learning models for the classification of technical inspection recommendations regarding the trend to increase criticality.

@inproceedings{SilvaMGEK23,
  author = {Silva, Luciana and Marques, Marcelo and Gomes, Marcos and Escovedo, Tatiana and Kalinowski, Marcos},
  title = {Inference of Properties of a Natural Gas Processing Plant Through the Application of Machine Learning to Time Series},
  year = {2023},
  isbn = {9798400707599},
  publisher = {Association for Computing Machinery},
  urlAuthor_version = {http://www.inf.puc-rio.br/~kalinowski/publications/SilvaMGEK23.pdf},
  doi = {10.1145/3592813.3592894},
  abstract = {Context: Virtual analyzers or inferences are mathematical models widely used in the chemical process industry, as they allow real-time prediction of properties of interest from measurements of basic properties available at the appropriate frequency. Problem: Therefore, they are a key component of chemical process control and optimization structures. Inferences with superior prediction capabilities provide a better process control reducing losses and improving profits. Solution: This work seeks to establish whether Machine Learning (ML) algorithms and techniques could be used to generate inferences for the ethane content in Liquefied Petroleum Gas (LPG) produced in a Natural Gas Processing Unit (NGPU) with advantages over the inferences traditionally used. IS Theory: This work is associated with the Theory of the knowledge-based company, assisting in decision-making and efficiency in the application of resources. Method: In order to accomplish this, a dataset with five years of real data from a Petrobras UPGN has been obtained, and models with different types of algorithms were generated. The machine learning models are compared between them and against models similar to the inferences currently used in the plant, in order to verify whether the ML tools have real potential to provide better results. Summary of Results: The obtained results showed that the machine learning models have a much better representation capability when compared to the traditional inference models. Contributions and Impact in the IS area: The paper contributes to the use of machine learning techniques to generate better inferences for the ethane content in LGP produced in a NGPU, making the analytical process more agile and assertive to increase the generation of value for Petrobras.},
  booktitle = {Proceedings of the XIX Brazilian Symposium on Information Systems},
  pages = {102–111},
  numpages = {10},
  location = {Macei\'{o}, Brazil},
  series = {SBSI '23}
}

Context: Virtual analyzers or inferences are mathematical models widely used in the chemical process industry, as they allow real-time prediction of properties of interest from measurements of basic properties available at the appropriate frequency. Problem: Therefore, they are a key component of chemical process control and optimization structures. Inferences with superior prediction capabilities provide a better process control reducing losses and improving profits. Solution: This work seeks to establish whether Machine Learning (ML) algorithms and techniques could be used to generate inferences for the ethane content in Liquefied Petroleum Gas (LPG) produced in a Natural Gas Processing Unit (NGPU) with advantages over the inferences traditionally used. IS Theory: This work is associated with the Theory of the knowledge-based company, assisting in decision-making and efficiency in the application of resources. Method: In order to accomplish this, a dataset with five years of real data from a Petrobras UPGN has been obtained, and models with different types of algorithms were generated. The machine learning models are compared between them and against models similar to the inferences currently used in the plant, in order to verify whether the ML tools have real potential to provide better results. Summary of Results: The obtained results showed that the machine learning models have a much better representation capability when compared to the traditional inference models. Contributions and Impact in the IS area: The paper contributes to the use of machine learning techniques to generate better inferences for the ethane content in LGP produced in a NGPU, making the analytical process more agile and assertive to increase the generation of value for Petrobras.

@inproceedings{VaqueiroVEK23,
  author = {Vaqueiro, Ramon and Vargas, Ana and Escovedo, Tatiana and Kalinowski, Marcos},
  title = {Machine Learning Applied to Open Government Data for the Detection of Improprieties in the Application of Public Resources},
  year = {2023},
  isbn = {9798400707599},
  publisher = {Association for Computing Machinery},
  urlAuthor_version = {http://www.inf.puc-rio.br/~kalinowski/publications/VaqueiroVEK23.pdf},
  doi = {10.1145/3592813.3592908},
  abstract = {Context: Making government data publicly available is an important mechanism of transparency and social control. In this regard, inumerous laws have made it mandatory to divulgate government procurement data. Problem: The large volume of unstructured textual information available on government portals is an obstacle to effective social control. Making it difficult to do more in-depth analyzes of public spending. Solution: Use of Machine Learning algorithms to perform text mining and grouping items acquired by public administration. Labeling public purchases and grouping similar items, in order to facilitate the detection of improprieties in government purchases. IS Theory: This work is associated with the Theory of Computational Learning, which aims to understand the fundamental principles of learning and design better-automated methods. Method: The article is a case study, and its evaluation was executed with the support of specialists in the field. The results were analyzed based on a quantitative approach. Summary of Results: The results observed in the evaluated cases were promising, the resulting clusters from the application of the solution had sufficiently coherent semantic values, in order to allow more complex analyzes of government purchases. Contributions and Impact in the IS area: The results show that applying text mining and machine learning techniques can extract useful information from government purchases data and allowing to perform better analyzes of public spending.},
  booktitle = {Proceedings of the XIX Brazilian Symposium on Information Systems},
  pages = {213–220},
  numpages = {8},
  keywords = {Text Mining, Public Purchases, Machine Learning.},
  location = {Macei\'{o}, Brazil},
  series = {SBSI '23}
}

Context: Making government data publicly available is an important mechanism of transparency and social control. In this regard, inumerous laws have made it mandatory to divulgate government procurement data. Problem: The large volume of unstructured textual information available on government portals is an obstacle to effective social control. Making it difficult to do more in-depth analyzes of public spending. Solution: Use of Machine Learning algorithms to perform text mining and grouping items acquired by public administration. Labeling public purchases and grouping similar items, in order to facilitate the detection of improprieties in government purchases. IS Theory: This work is associated with the Theory of Computational Learning, which aims to understand the fundamental principles of learning and design better-automated methods. Method: The article is a case study, and its evaluation was executed with the support of specialists in the field. The results were analyzed based on a quantitative approach. Summary of Results: The results observed in the evaluated cases were promising, the resulting clusters from the application of the solution had sufficiently coherent semantic values, in order to allow more complex analyzes of government purchases. Contributions and Impact in the IS area: The results show that applying text mining and machine learning techniques can extract useful information from government purchases data and allowing to perform better analyzes of public spending.

@inproceedings{FernandesK23,
  author    = {Eduardo Fernandes and Marcos Kalinowski},
  title     = {On the perceived relevance of critical internal quality attributes when evolving software features},
  abstract  = {Several refactorings performed while evolving software features aim to improve internal quality attributes like cohesion and complexity. Indeed, internal attributes can become critical if their measurements assume anomalous values. Yet, current knowledge is scarce on how developers perceive the relevance of critical internal attributes while evolving features. This qualitative study investigates the developers’ perception of the relevance of critical internal attributes when evolving features. We target six class-level critical attributes: low cohesion, high complexity, high coupling, large hierarchy depth, large hierarchy breadth, and large size. We performed two industrial case studies based on online focus group sessions. Developers discussed how much (and why) critical attributes are relevant when adding or enhancing features. We assessed the relevance of critical attributes individually and relatively, the reasons behind the relevance of each critical attribute, and the interrelations of critical attributes. Low cohesion and high complexity were perceived as very relevant because they often make evolving features hard while tracking failures and adding features. The other critical attributes were perceived as less relevant when reusing code or adopting design patterns. An example of perceived interrelation is high complexity leading to high coupling.},
  booktitle = {International Conference on Human Aspects in Software Engineering, {CHASE} 2023, Melbourne, Australia, May 14-15},
  pages     = {1-12},
  year      = {2023},
  urlAuthor_version = {http://www.inf.puc-rio.br/~kalinowski/publications/FernandesK23.pdf},
  doi       = {},
}

Several refactorings performed while evolving software features aim to improve internal quality attributes like cohesion and complexity. Indeed, internal attributes can become critical if their measurements assume anomalous values. Yet, current knowledge is scarce on how developers perceive the relevance of critical internal attributes while evolving features. This qualitative study investigates the developers’ perception of the relevance of critical internal attributes when evolving features. We target six class-level critical attributes: low cohesion, high complexity, high coupling, large hierarchy depth, large hierarchy breadth, and large size. We performed two industrial case studies based on online focus group sessions. Developers discussed how much (and why) critical attributes are relevant when adding or enhancing features. We assessed the relevance of critical attributes individually and relatively, the reasons behind the relevance of each critical attribute, and the interrelations of critical attributes. Low cohesion and high complexity were perceived as very relevant because they often make evolving features hard while tracking failures and adding features. The other critical attributes were perceived as less relevant when reusing code or adopting design patterns. An example of perceived interrelation is high complexity leading to high coupling.

@book{KalinowskiEVL23,
  title={Engenharia de Software para Ci{\^e}ncia de Dados: Um guia de boas pr{\'a}ticas com {\^e}nfase na constru{\c{c}}{\~a}o de sistemas de Machine Learning em Python},
  author={Kalinowski, Marcos and Escovedo, Tatiana and Villamizar, Hugo and Lopes, H{\'e}lio},
  abstract  = {Os avan{\c{c}}os recentes na {\'a}rea de Ci{\^e}ncia de Dados — em especial em Machine Learning — e a disponibilidade de grandes quantidades de dados t{\^e}m tornado vi{\'a}vel e de relev{\^a}ncia pr{\'a}tica a incorpora{\c{c}}{\~a}o de componentes de Machine Learning em sistemas de software. Esses componentes permitem às solu{\c{c}}ões de software aprender a partir dos dados e realizar predi{\c{c}}ões inteligentes. Entretanto, sistemas que envolvem este tipo de componente muitas vezes s{\~a}o construídos sem considerar boas pr{\'a}ticas da {\'a}rea de Engenharia de Software, levando muitos projetos de Ci{\^e}ncia de Dados ao fracasso, com solu{\c{c}}ões que acabam n{\~a}o sendo implantadas na pr{\'a}tica por n{\~a}o atenderem às necessidades dos clientes. Com isso em vista, este livro foi cuidadosamente elaborado a partir de evid{\^e}ncias científicas e experi{\^e}ncias pr{\'a}ticas dos autores em diversos projetos. O objetivo desta obra {\'e} estabelecer uma refer{\^e}ncia sobre Engenharia de Software para Ci{\^e}ncia de Dados, compartilhando conhecimento (at{\'e} ent{\~a}o n{\~a}o consolidado) para capacitar profissionais interessados ou atuantes em Ci{\^e}ncia de Dados na constru{\c{c}}{\~a}o de sistemas baseados em Machine Learning, mostrando como construir esses sistemas end-to-end, adaptando e aplicando as melhores pr{\'a}ticas da Engenharia de Software para esse contexto. Voc{\^e} aprender{\'a} a aplicar abordagens {\'a}geis para a engenharia de sistemas inteligentes e a especificar e desenvolver sistemas baseados em Machine Learning na pr{\'a}tica em Python, utilizando os principais algoritmos de classifica{\c{c}}{\~a}o e regress{\~a}o, seguindo princípios de projeto e boas pr{\'a}ticas de codifica{\c{c}}{\~a}o. Voc{\^e} aprender{\'a}, ainda, a realizar o controle da qualidade de sistemas inteligentes, conhecer{\'a} alternativas para a arquitetura desses sistemas e diferentes formas de implanta{\c{c}}{\~a}o de modelos. Por fim, conhecer{\'a} conceitos de ger{\^e}ncia de configura{\c{c}}{\~a}o, DevOps e MLOps, comumente empregados nesse tipo de projeto. O livro compila a vasta experi{\^e}ncia dos autores envolvendo a entrega de dezenas de projetos de sistemas de software inteligentes na pr{\'a}tica para diversas empresas, realizados por meio da iniciativa ExACTa PUC-Rio, incluindo solu{\c{c}}ões com pedidos de patente depositados e premiados (Reconhecimento à Inova{\c{c}}{\~a}o CENPES 2021, Inventor Petrobras 2022, entre outros). Os autores s{\~a}o pesquisadores consolidados nas {\'a}reas de Engenharia de Software e Ci{\^e}ncia de Dados e pioneiros na interse{\c{c}}{\~a}o entre essas {\'a}reas, sendo protagonistas no estabelecimento da Engenharia de Software para Ci{\^e}ncia de Dados no Brasil. Em 2021, os autores criaram o primeiro curso de extens{\~a}o em Engenharia de Software para Ci{\^e}ncia de Dados do país, formando centenas de alunos em diversas turmas oferecidas pela PUC-Rio, tanto abertas quanto in-company. Atualmente, a disciplina faz parte do currículo das pós-gradua{\c{c}}ões lato sensu em Ci{\^e}ncia de Dados e Analytics e em Engenharia de Software e do elenco de disciplinas da pós-gradua{\c{c}}{\~a}o stricto sensu (mestrado e doutorado) do Departamento de Inform{\'a}tica da PUC-Rio. Al{\'e}m de auxiliar profissionais da {\'a}rea, este livro pode ser utilizado como refer{\^e}ncia bibliogr{\'a}fica para disciplinas com propósito similar em outras institui{\c{c}}ões.},
  note = {<font color="red">Finalist for Brazil's most prestigious academic literature award - <i>Jabuti Acad&ecirc;mico</i> - by the Brazilian Book Chamber (C&acirc;mara Brasileira do Livro).</font>},
  pages     = {1-476},
  year={2023},
  publisher={Casa do C{\'o}digo},
  url = {https://www.casadocodigo.com.br/products/livro-escd}
}

Os avanços recentes na área de Ciência de Dados — em especial em Machine Learning — e a disponibilidade de grandes quantidades de dados têm tornado viável e de relevância prática a incorporação de componentes de Machine Learning em sistemas de software. Esses componentes permitem às soluções de software aprender a partir dos dados e realizar predições inteligentes. Entretanto, sistemas que envolvem este tipo de componente muitas vezes são construídos sem considerar boas práticas da área de Engenharia de Software, levando muitos projetos de Ciência de Dados ao fracasso, com soluções que acabam não sendo implantadas na prática por não atenderem às necessidades dos clientes. Com isso em vista, este livro foi cuidadosamente elaborado a partir de evidências científicas e experiências práticas dos autores em diversos projetos. O objetivo desta obra é estabelecer uma referência sobre Engenharia de Software para Ciência de Dados, compartilhando conhecimento (até então não consolidado) para capacitar profissionais interessados ou atuantes em Ciência de Dados na construção de sistemas baseados em Machine Learning, mostrando como construir esses sistemas end-to-end, adaptando e aplicando as melhores práticas da Engenharia de Software para esse contexto. Você aprenderá a aplicar abordagens ágeis para a engenharia de sistemas inteligentes e a especificar e desenvolver sistemas baseados em Machine Learning na prática em Python, utilizando os principais algoritmos de classificação e regressão, seguindo princípios de projeto e boas práticas de codificação. Você aprenderá, ainda, a realizar o controle da qualidade de sistemas inteligentes, conhecerá alternativas para a arquitetura desses sistemas e diferentes formas de implantação de modelos. Por fim, conhecerá conceitos de gerência de configuração, DevOps e MLOps, comumente empregados nesse tipo de projeto. O livro compila a vasta experiência dos autores envolvendo a entrega de dezenas de projetos de sistemas de software inteligentes na prática para diversas empresas, realizados por meio da iniciativa ExACTa PUC-Rio, incluindo soluções com pedidos de patente depositados e premiados (Reconhecimento à Inovação CENPES 2021, Inventor Petrobras 2022, entre outros). Os autores são pesquisadores consolidados nas áreas de Engenharia de Software e Ciência de Dados e pioneiros na interseção entre essas áreas, sendo protagonistas no estabelecimento da Engenharia de Software para Ciência de Dados no Brasil. Em 2021, os autores criaram o primeiro curso de extensão em Engenharia de Software para Ciência de Dados do país, formando centenas de alunos em diversas turmas oferecidas pela PUC-Rio, tanto abertas quanto in-company. Atualmente, a disciplina faz parte do currículo das pós-graduações lato sensu em Ciência de Dados e Analytics e em Engenharia de Software e do elenco de disciplinas da pós-graduação stricto sensu (mestrado e doutorado) do Departamento de Informática da PUC-Rio. Além de auxiliar profissionais da área, este livro pode ser utilizado como referência bibliográfica para disciplinas com propósito similar em outras instituições.

@Inbook{FurtadoVLSKO23,
  author="Furtado, Viviane R. and Vignando, Henrique and Luz, Carlos D. and Steinmacher, Igor F. and Kalinowski, Marcos and OliveiraJr, Edson",
  editor="OliveiraJr, Edson",
  title="Controlled Experimentation of Software Product Lines",
  bookTitle="UML-Based Software Product Line Engineering with SMarty",
  year="2023",
  publisher="Springer International Publishing",
  address="Cham",
  pages="417--443",
  abstract="The process of experimentation is one of several scientific methods that can provide evidence for a proof of a theory. This process is counterpoint to the real world observation method, thus providing a reliable body of knowledge. However, in the experimentation for emerging areas and in the consolidation process in scientific and industrial communities, such as the software product line (SPL), there has been a constant lack of adequate documentation of experiments that makes it difficult to repeat, replicate, and reproduce studies in SPL. Therefore, this chapter presents a set of guidelines for the quality assessment of SPL experiments with its conceptual model to support the understanding of the proposed guidelines, as well as an ontology for SPL experiments, called OntoExper-SPL, in addition to support the teaching experimentation in SPL. Thus, these points aim to improve the planning, conduction, analysis, sharing, and documentation of SPL experiments, supporting the construction of a reliable and reference body of knowledge in such a context in addition to enabling improvement in the teaching of SPL experiments.",
  isbn="978-3-031-18556-4",
  doi="10.1007/978-3-031-18556-4_19",
  urlAuthor_version = {http://www.inf.puc-rio.br/~kalinowski/publications/FurtadoVLSKO23.pdf}  
}

The process of experimentation is one of several scientific methods that can provide evidence for a proof of a theory. This process is counterpoint to the real world observation method, thus providing a reliable body of knowledge. However, in the experimentation for emerging areas and in the consolidation process in scientific and industrial communities, such as the software product line (SPL), there has been a constant lack of adequate documentation of experiments that makes it difficult to repeat, replicate, and reproduce studies in SPL. Therefore, this chapter presents a set of guidelines for the quality assessment of SPL experiments with its conceptual model to support the understanding of the proposed guidelines, as well as an ontology for SPL experiments, called OntoExper-SPL, in addition to support the teaching experimentation in SPL. Thus, these points aim to improve the planning, conduction, analysis, sharing, and documentation of SPL experiments, supporting the construction of a reliable and reference body of knowledge in such a context in addition to enabling improvement in the teaching of SPL experiments.

@ARTICLE{KuhrmannEtAl22,
  author={Kuhrmann, Marco and Tell, Paolo and Hebig, Regina and Klunder, Jil Ann-Christin and Munch, Jurgen and Linssen, Oliver and Pfahl, Dietmar and Felderer, Michael and Prause, Christian and Macdonell, Steve and Nakatumba-Nabende, Joyce and Raffo, David and Beecham, Sarah and Tuzun, Eray and Lopez, Gustavo and Paez, Nicolas and Fontdevila, Diego and Licorish, Sherlock and Kupper, Steffen and Ruhe, Guenther and Knauss, Eric and Ozcan-Top, Ozden and Clarke, Paul and Mc Caffery, Fergal Hugh and Genero, Marcela and Vizcaino, Aurora and Piattini, Mario and Kalinowski, Marcos and Conte, Tayana and Prikladnicki, Rafael and Krusche, Stephan and Coskuncay, Ahmet and Scott, Ezequiel and Calefato, Fabio and Pimonova, Svetlana and Pfeiffer, Rolf-Helge and Pagh Schultz, Ulrik and Heldal, Rogardt and Fazal-Baqaie, Masud and Anslow, Craig and Nayebi, Maleknaz and Schneider, Kurt and Sauer, Stefan and Winkler, Dietmar and Biffl, Stefan and Bastarrica, Cecilia and Richardson, Ita},
  journal={IEEE Transactions on Software Engineering}, 
  title={What Makes Agile Software Development Agile}, 
  year={2022},
  volume={48},
  number={9},
  pages={3523-3539},
  urlAuthor_version = {http://www.inf.puc-rio.br/~kalinowski/publications/KuhrmannEtAl22.pdf},
  doi={10.1109/TSE.2021.3099532},
  keywords={Agile Software Development, Hybrid Development Methods, Survey Research, Software Development, Software Process},
  abstract={Together with many success stories, promises such as the increase in production speed and the improvement in stakeholders' collaboration have contributed to making agile a transformation in the software industry in which many companies want to take part. However, driven either by a natural and expected evolution or by contextual factors that challenge the adoption of agile methods as prescribed by their creator(s), software processes in practice mutate into hybrids over time. Are these still agile In this article, we investigate the question: what makes a software development method agile We present an empirical study grounded in a large-scale international survey that aims to identify software development methods and practices that improve or tame agility. Based on 556 data points, we analyze the perceived degree of agility in the implementation of standard project disciplines and its relation to used development methods and practices. Our findings suggest that only a small number of participants operate their projects in a purely traditional or agile manner (under 15%). That said, most project disciplines and most practices show a clear trend towards increasing degrees of agility. Compared to the methods used to develop software, the selection of practices has a stronger effect on the degree of agility of a given discipline. Finally, there are no methods or practices that explicitly guarantee or prevent agility. We conclude that agility cannot be defined solely at the process level. Additional factors need to be taken into account when trying to implement or improve agility in a software company. Finally, we discuss the field of software process-related research in the light of our findings and present a roadmap for future research.}
}

Together with many success stories, promises such as the increase in production speed and the improvement in stakeholders' collaboration have contributed to making agile a transformation in the software industry in which many companies want to take part. However, driven either by a natural and expected evolution or by contextual factors that challenge the adoption of agile methods as prescribed by their creator(s), software processes in practice mutate into hybrids over time. Are these still agile In this article, we investigate the question: what makes a software development method agile We present an empirical study grounded in a large-scale international survey that aims to identify software development methods and practices that improve or tame agility. Based on 556 data points, we analyze the perceived degree of agility in the implementation of standard project disciplines and its relation to used development methods and practices. Our findings suggest that only a small number of participants operate their projects in a purely traditional or agile manner (under 15%). That said, most project disciplines and most practices show a clear trend towards increasing degrees of agility. Compared to the methods used to develop software, the selection of practices has a stronger effect on the degree of agility of a given discipline. Finally, there are no methods or practices that explicitly guarantee or prevent agility. We conclude that agility cannot be defined solely at the process level. Additional factors need to be taken into account when trying to implement or improve agility in a software company. Finally, we discuss the field of software process-related research in the light of our findings and present a roadmap for future research.

@article{WohlinKFM22,
  title = {Successful combination of database search and snowballing for identification of primary studies in systematic literature studies},
  journal = {Information and Software Technology},
  volume = {147},
  pages = {106908:1-106908:12},
  year = {2022},
  issn = {0950-5849},
  doi = {https://doi.org/10.1016/j.infsof.2022.106908},
  urlAuthor_version = {http://www.inf.puc-rio.br/~kalinowski/publications/WohlinKFM22.pdf},
  author = {Claes Wohlin and Marcos Kalinowski and Katia {Romero Felizardo} and Emilia Mendes},
  keywords = {Systematic literature reviews, Hybrid search, Snowballing, Scopus},
  abstract = {Background: A good search strategy is essential for a successful systematic literature study. Historically, database searches have been the norm, which was later complemented with snowball searches. Our conjecture is that we can perform even better searches if combining these two search approaches, referred to as a hybrid search strategy. Objective: Our main objective was to compare and evaluate a hybrid search strategy. Furthermore, we compared four alternative hybrid search strategies to assess whether we could identify more cost-efficient ways of searching for relevant primary studies. Methods: To compare and evaluate the hybrid search strategy, we replicated the search procedure in a systematic literature review (SLR) on industry–academia collaboration in software engineering. The SLR used a more “traditional” approach to searching for relevant articles for an SLR, while our replication was executed using a hybrid search strategy. Results: In our evaluation, the hybrid search strategy was superior in identifying relevant primary studies. It identified 30% more primary studies and even more studies when focusing only on peer-reviewed articles. To embrace individual viewpoints when assessing research articles and minimise the risk of missing primary studies, we introduced two new concepts, wild cards and borderline articles, when performing systematic literature studies. Conclusions: The hybrid search strategy is a strong contender for being used when performing systematic literature studies. Furthermore, alternative hybrid search strategies may be viable if selected wisely in relation to the start set for snowballing. Finally, the two new concepts were judged as essential to cater for different individual judgements and to minimise the risk of excluding primary studies that ought to be included.}
}

Background: A good search strategy is essential for a successful systematic literature study. Historically, database searches have been the norm, which was later complemented with snowball searches. Our conjecture is that we can perform even better searches if combining these two search approaches, referred to as a hybrid search strategy. Objective: Our main objective was to compare and evaluate a hybrid search strategy. Furthermore, we compared four alternative hybrid search strategies to assess whether we could identify more cost-efficient ways of searching for relevant primary studies. Methods: To compare and evaluate the hybrid search strategy, we replicated the search procedure in a systematic literature review (SLR) on industry–academia collaboration in software engineering. The SLR used a more “traditional” approach to searching for relevant articles for an SLR, while our replication was executed using a hybrid search strategy. Results: In our evaluation, the hybrid search strategy was superior in identifying relevant primary studies. It identified 30% more primary studies and even more studies when focusing only on peer-reviewed articles. To embrace individual viewpoints when assessing research articles and minimise the risk of missing primary studies, we introduced two new concepts, wild cards and borderline articles, when performing systematic literature studies. Conclusions: The hybrid search strategy is a strong contender for being used when performing systematic literature studies. Furthermore, alternative hybrid search strategies may be viable if selected wisely in relation to the start set for snowballing. Finally, the two new concepts were judged as essential to cater for different individual judgements and to minimise the risk of excluding primary studies that ought to be included.

@article{MendoncaK22,
  title = {An empirical investigation on the challenges of creating custom static analysis rules for defect localization},
  journal = {Software Quality Journal},
  volume = {},
  pages = {1-28},
  year = {2022},
  issn = {0164-1212},
  doi = {https://doi.org/10.1007/s11219-021-09580-z},
  urlAuthor_version = {http://www.inf.puc-rio.br/~kalinowski/publications/MendoncaK22.pdf},
  author = {Diogo S. Mendon{\c{c}}a and Marcos Kalinowski},
  keywords = {Custom static analysis rules, Code defect patterns, Pattern-Driven Maintenance, Observational study},
  abstract = {Custom static analysis rules, i.e., rules specific for one or more applications, have been successfully applied to perform corrective and preventive software maintenance. Pattern-driven maintenance (PDM) is a method designed to support the creation of such rules during software maintenance. However, as PDM was recently proposed, few maintainers have reported on its usage. Hence, the challenges and skills needed to apply PDM properly are unknown. In this paper, we investigate the challenges faced by maintainers on applying PDM for creating custom static analysis rules for defect localization. We conducted an observational study on novice maintainers creating custom static analysis rules by applying PDM. The study was divided into three tasks: (i) identifying a defect pattern, (ii) programming a static analysis rule to locate instances of the pattern, and (iii) verifying the located instances. We analyzed the efficiency and acceptance of maintainers on applying PDM and their comments on task challenges. We observed that previous knowledge on debugging, the subject software, and related technologies influenced the performance of maintainers as well as the time to learn the technology involved in rule programming. The results strengthen our confidence that PDM can help maintainers in producing custom static analysis rules for locating defects. However, a proper selection and training of maintainers is needed to apply PDM effectively. Also, using a higher level of abstraction can ease static analysis rule programming for novice maintainers.}
}

Custom static analysis rules, i.e., rules specific for one or more applications, have been successfully applied to perform corrective and preventive software maintenance. Pattern-driven maintenance (PDM) is a method designed to support the creation of such rules during software maintenance. However, as PDM was recently proposed, few maintainers have reported on its usage. Hence, the challenges and skills needed to apply PDM properly are unknown. In this paper, we investigate the challenges faced by maintainers on applying PDM for creating custom static analysis rules for defect localization. We conducted an observational study on novice maintainers creating custom static analysis rules by applying PDM. The study was divided into three tasks: (i) identifying a defect pattern, (ii) programming a static analysis rule to locate instances of the pattern, and (iii) verifying the located instances. We analyzed the efficiency and acceptance of maintainers on applying PDM and their comments on task challenges. We observed that previous knowledge on debugging, the subject software, and related technologies influenced the performance of maintainers as well as the time to learn the technology involved in rule programming. The results strengthen our confidence that PDM can help maintainers in producing custom static analysis rules for locating defects. However, a proper selection and training of maintainers is needed to apply PDM effectively. Also, using a higher level of abstraction can ease static analysis rule programming for novice maintainers.

@article{LaignerEtAl22,
  title = {Cataloging dependency injection anti-patterns in software systems},
  journal = {Journal of Systems and Software},
  volume = {184},
  pages = {1-31},
  year = {2022},
  issn = {0164-1212},
  doi = {https://doi.org/10.1016/j.jss.2021.111125},
  urlAuthor_version = {http://www.inf.puc-rio.br/~kalinowski/publications/LaignerEtAl22.pdf},
  author = {Rodrigo Laigner and Diogo Mendon{\c{c}}a and Alessandro Garcia and Marcos Kalinowski},
  keywords = {Dependency injection, Dependency inversion, Inversion of control, Anti-pattern},
  abstract = {Context: Dependency Injection (DI) is a commonly applied mechanism to decouple classes from their dependencies in order to provide higher modularization. However, bad DI practices often lead to negative consequences, such as increasing coupling. Although white literature conjectures about the existence of DI anti-patterns, there is no evidence on their practical relevance, usefulness, and generality. Objective: The objective of this study is to propose and evaluate a catalog of DI anti-patterns and associated refactorings. Methodology: We reviewed existing reported DI anti-patterns in order to analyze their completeness. The limitations found in literature motivated proposing a novel catalog of 12 DI anti-patterns. We developed a tool to statically analyze the occurrence level of the candidate DI anti-patterns in both open-source and industry projects. Next, we survey practitioners to assess their perception on the relevance, usefulness, and their willingness on refactoring anti-pattern instances of the catalog. Results: Our static code analyzer tool showed a relative recall of 92.19% and high average precision. It revealed that at least 9 different DI anti-patterns appeared frequently in the analyzed projects. Besides, our survey confirmed the perceived relevance of the catalog and developers expressed their willingness to refactor instances of anti-patterns from source code. Conclusion: The catalog contains DI anti-patterns that occur in practice and that are perceived as useful. Sharing it with practitioners may help them to avoid such anti-patterns, thus improving source-code quality.}
}

Context: Dependency Injection (DI) is a commonly applied mechanism to decouple classes from their dependencies in order to provide higher modularization. However, bad DI practices often lead to negative consequences, such as increasing coupling. Although white literature conjectures about the existence of DI anti-patterns, there is no evidence on their practical relevance, usefulness, and generality. Objective: The objective of this study is to propose and evaluate a catalog of DI anti-patterns and associated refactorings. Methodology: We reviewed existing reported DI anti-patterns in order to analyze their completeness. The limitations found in literature motivated proposing a novel catalog of 12 DI anti-patterns. We developed a tool to statically analyze the occurrence level of the candidate DI anti-patterns in both open-source and industry projects. Next, we survey practitioners to assess their perception on the relevance, usefulness, and their willingness on refactoring anti-pattern instances of the catalog. Results: Our static code analyzer tool showed a relative recall of 92.19% and high average precision. It revealed that at least 9 different DI anti-patterns appeared frequently in the analyzed projects. Besides, our survey confirmed the perceived relevance of the catalog and developers expressed their willingness to refactor instances of anti-patterns from source code. Conclusion: The catalog contains DI anti-patterns that occur in practice and that are perceived as useful. Sharing it with practitioners may help them to avoid such anti-patterns, thus improving source-code quality.

@article{NascimentoEK22, 
  title={Construction of Mortality Tables using Bidirectional LSTM Neural Networks to Predict Death Probabilities}, 
  volume={15}, 
  urlAuthor_version = {http://www.inf.puc-rio.br/~kalinowski/publications/NascimentoEK22.pdf},  
  DOI={10.5753/isys.2022.2230}, 
  abstract={Mortality Tables are tables structured with mortality data, especially mortality rates observed at all ages, used in pension funds and life insurance markets. This article concerns the application of the neural network model to the construction of future mortality tables, using the Lee-Carter model for comparison. The proposed model was a LSTM (Long-Short Term Memory) Neural Network model, including a bidirectional variation. This network is characterized by the sequential processing of data over time. The data for the prediction came from historical mortality table data prepared by the IBGE (Brazilian Institute of Geography and Statistics) and the Human Mortality Database. The results point to a reasonable use as an auxiliary tool for predicting death probabilities.}, 
  number={1}, 
  journal={iSys - Brazilian Journal of Information Systems}, 
  author={Nascimento, Jos{\'e} Douglas and Escovedo, Tatiana and Kalinowski, Marcos},
  year={2022}, 
  month={Jul.}, 
  pages={9:1–9:24} 
}

Mortality Tables are tables structured with mortality data, especially mortality rates observed at all ages, used in pension funds and life insurance markets. This article concerns the application of the neural network model to the construction of future mortality tables, using the Lee-Carter model for comparison. The proposed model was a LSTM (Long-Short Term Memory) Neural Network model, including a bidirectional variation. This network is characterized by the sequential processing of data over time. The data for the prediction came from historical mortality table data prepared by the IBGE (Brazilian Institute of Geography and Statistics) and the Human Mortality Database. The results point to a reasonable use as an auxiliary tool for predicting death probabilities.

@article{BarchilonEK22, 
  title = {Machine Learning Applied to the INSS Benefit Request - Extended Analysis},
  volume={15},
  urlAuthor_version = {http://www.inf.puc-rio.br/~kalinowski/publications/BarchilonEK22.pdf},  
  DOI={10.5753/isys.2022.2224},
  abstract={The materialization of social protection, foreseen in the Brazilian Constitution’s Social Security chapter, specifically in the scope of Welfare, occurs through the granting and maintenance of benefits to all Brazilians who need this protection This right generates a huge demand of millions of requests for annual benefits to the INSS (National Institute of Social Security), which is the operator of these services. Receiving and analyzing benefit requests, in a timely manner and with assertiveness, is complex and challenging. The volume of millions of applications for benefits annually, the diversity of benefits available, different criteria for granting and the urgency that the nature of these applications requires for the maintenance of life for applicants, express this complex and challenging environment. Within this context, the present study aims to develop some models, using machine learning techniques, and select the best one, which can predict whether a certain benefit request will be granted or rejected. This prediction would help in the analysis of new benefit requests, making room for the dynamics of the analysis process to be directed more quickly and assertively. The data source for the construction of the models in this work was obtained from the INSS Open Data Portal, which are included in the INSS Open Data Plan. This dataset is composed of monthly files of Decided Benefits (Granted and Dismissed) from December 2018 to June 2020. As a scope of analysis, algorithms such as KNN, SVC, Decision Trees, Logistic Regression, etc. were addressed. Models were also built using the Ensemble Bagging and Boosting techniques, reaching a set of seventeen analyzed algorithms. The algorithm that obtained the best performance, using the F1 metric as the determinant, was the eXtreme Gradient Boosting (XGB) Classifier with 80%. With this, the model performs the prediction with approximately 84% Accuracy, 76% Sensitivity and 81% for AUC (Area Under the Curve). As a result of the study, a model capable of predicting whether a given benefit requirement would be granted or denied was obtained, based on the requirement data, with a performance within the expectations established in the objectives. Therefore, this article analyzed seventeen machine learning algorithms with the aim of building a model to predict whether a given benefit request to the INSS will be granted or rejected. As a contribution of this article to the IS, we highlight the initiative to apply a machine learning model in a domain not yet explored by other research, using, as a data source, open data made available by the federal government in the cloud. It is hoped that the result of this study will open space for new scientific research in the field of machine learning to be developed in this domain, with the aim of helping with real problems in this sensitive part of the lives of Brazilians and of public administration.},
  number={1},
  journal={iSys - Brazilian Journal of Information Systems},
  author={Barchilon, Ney and Escovedo, Tatiana and Kalinowski, Marcos}, 
  year={2022}, 
  month={Jul.}, 
  pages={5:1–5:28} 
}

The materialization of social protection, foreseen in the Brazilian Constitution’s Social Security chapter, specifically in the scope of Welfare, occurs through the granting and maintenance of benefits to all Brazilians who need this protection This right generates a huge demand of millions of requests for annual benefits to the INSS (National Institute of Social Security), which is the operator of these services. Receiving and analyzing benefit requests, in a timely manner and with assertiveness, is complex and challenging. The volume of millions of applications for benefits annually, the diversity of benefits available, different criteria for granting and the urgency that the nature of these applications requires for the maintenance of life for applicants, express this complex and challenging environment. Within this context, the present study aims to develop some models, using machine learning techniques, and select the best one, which can predict whether a certain benefit request will be granted or rejected. This prediction would help in the analysis of new benefit requests, making room for the dynamics of the analysis process to be directed more quickly and assertively. The data source for the construction of the models in this work was obtained from the INSS Open Data Portal, which are included in the INSS Open Data Plan. This dataset is composed of monthly files of Decided Benefits (Granted and Dismissed) from December 2018 to June 2020. As a scope of analysis, algorithms such as KNN, SVC, Decision Trees, Logistic Regression, etc. were addressed. Models were also built using the Ensemble Bagging and Boosting techniques, reaching a set of seventeen analyzed algorithms. The algorithm that obtained the best performance, using the F1 metric as the determinant, was the eXtreme Gradient Boosting (XGB) Classifier with 80%. With this, the model performs the prediction with approximately 84% Accuracy, 76% Sensitivity and 81% for AUC (Area Under the Curve). As a result of the study, a model capable of predicting whether a given benefit requirement would be granted or denied was obtained, based on the requirement data, with a performance within the expectations established in the objectives. Therefore, this article analyzed seventeen machine learning algorithms with the aim of building a model to predict whether a given benefit request to the INSS will be granted or rejected. As a contribution of this article to the IS, we highlight the initiative to apply a machine learning model in a domain not yet explored by other research, using, as a data source, open data made available by the federal government in the cloud. It is hoped that the result of this study will open space for new scientific research in the field of machine learning to be developed in this domain, with the aim of helping with real problems in this sensitive part of the lives of Brazilians and of public administration.

@inproceedings{BravoSELK22,
  author    = {Fl{\'a}via Bravo and Luciana Sousa and Tatiana Escovedo and Helio Lopes and Marcos Kalinowski},
  title     = {Machine Learning Aplicado à Predi{\c{c}}{\~a}o da Obriga{\c{c}}{\~a}o de Investimento em P, D & I},
  abstract  = {Investments in Research, Development, and Innovation (R&D&I) from Brazil's oil and gas sector are substantial due to the obligation established by the National Agency of Petroleum, Natural Gas and Biofuels (ANP). Identifying the expectation of funding in an agile and simple way enables better planning, increasing the effectiveness of expenditures. This article proposes elaborating a machine learning model to predict the potential of mandatory investments that companies in the oil and gas sector must make in R&D&I, allowing better planning of the application of financial resources for universities and science and technology institutes.},
  booktitle = {Brazilian Symposium on Databases, {SBBD} 2022, Buzios, Brazil, Sept 19-23},
  pages     = {1-13},
  year      = {2022},
  urlAuthor_version = {http://www.inf.puc-rio.br/~kalinowski/publications/BravoSELK22.pdf},
  doi       = {10.5753/sbbd.2022.224344}
}

Investments in Research, Development, and Innovation (R&D&I) from Brazil's oil and gas sector are substantial due to the obligation established by the National Agency of Petroleum, Natural Gas and Biofuels (ANP). Identifying the expectation of funding in an agile and simple way enables better planning, increasing the effectiveness of expenditures. This article proposes elaborating a machine learning model to predict the potential of mandatory investments that companies in the oil and gas sector must make in R&D&I, allowing better planning of the application of financial resources for universities and science and technology institutes.

@inproceedings{CastroSELK22,
  author    = {Marcos Alexandre Castro and Nelson Souza Jr and Tatiana Escovedo and Helio Lopes and Marcos Kalinowski},
  title     = {Minera{\c{c}}{\~a}o de Processos Aplicada à Auditoria Interna na Marinha do Brasil},
  abstract  = {With society's growing interest in the application of public resources, the governmental internal audit activity gains importance. However, due to the growing integration and complexity of information systems, which process large amounts of data, auditors face new challenges when analyzing internal controls. This article applies the process mining technique to identify bottlenecks and control violations in internal processes, generating a new type of audit evidence. The proposed methodology was successfully applied in the processes of the Brazilian Navy, providing means to identify deviations and possible reductions of interactions and iterations performed by agents.},
  booktitle = {Brazilian Symposium on Databases, {SBBD} 2022, Buzios, Brazil, Sept 19-23},
  pages     = {1-13},
  year      = {2022},
  urlAuthor_version = {http://www.inf.puc-rio.br/~kalinowski/publications/CastroSELK22.pdf},  
  doi       = {10.5753/sbbd.2022.224346}
}

With society's growing interest in the application of public resources, the governmental internal audit activity gains importance. However, due to the growing integration and complexity of information systems, which process large amounts of data, auditors face new challenges when analyzing internal controls. This article applies the process mining technique to identify bottlenecks and control violations in internal processes, generating a new type of audit evidence. The proposed methodology was successfully applied in the processes of the Brazilian Navy, providing means to identify deviations and possible reductions of interactions and iterations performed by agents.

@inproceedings{LucchesiEK22,
  author    = {La{\'e}rcio Lucchesi and Tatiana Escovedo and Marcos Kalinowski},
  title     = {Padroniza{\c{c}}{\~a}o da Descri{\c{c}}{\~a}o de Produtos Comerciais utilizando NER},
  abstract  = {Product descriptions of retail establishments are information used in market analysis, but normally, these descriptions are poorly structured, nonstandardized, and vary a lot for the same product. This article proposes the use of natural language processing techniques, more specifically, a branch known as NER (Named Entity Recognition), to solve the automatic generation of standardized descriptions from retail product descriptions. As a result, the trained model proved to be adequate to extract the characteristic information of new products launched on the market and the consequent construction of their standardized descriptions.},
  booktitle = {Brazilian Symposium on Databases, {SBBD} 2022, Buzios, Brazil, Sept 19-23},
  pages     = {1-6},
  year      = {2022},
  urlAuthor_version = {http://www.inf.puc-rio.br/~kalinowski/publications/LucchesiEK22.pdf},  
  doi       = {10.5753/sbbd.2022.224347}
}

Product descriptions of retail establishments are information used in market analysis, but normally, these descriptions are poorly structured, nonstandardized, and vary a lot for the same product. This article proposes the use of natural language processing techniques, more specifically, a branch known as NER (Named Entity Recognition), to solve the automatic generation of standardized descriptions from retail product descriptions. As a result, the trained model proved to be adequate to extract the characteristic information of new products launched on the market and the consequent construction of their standardized descriptions.

@inproceedings{NapoleaoPHK22,
  author    = {Bianca Minetto Napole{\~a}o and Fabio Petrillo and Sylvain Hall{\'e} and Marcos Kalinowski},
  title     = {Towards Continuous Systematic Literature Review in Software Engineering},
  abstract  = {Context: New scientific evidence continuously arises with advances in Software Engineering (SE) research. Conventionally, Systematic Literature Reviews (SLRs) are not updated or updated intermittently, leaving gaps between updates, during which time the SLR may be missing crucial new evidence. Goal: We propose and evaluate a concept and process called Continuous Systematic Literature Review (CSLR) in SE. Method: To elaborate on the CSLR concept and process, we performed a synthesis of evidence by conducting a meta-ethnography, addressing knowledge from varied research areas. Furthermore, we conducted a case study to evaluate the CSLR process. Results: We describe the resulting CSLR process in BPMN format. The case study results provide indications on the importance and feasibility of applying CSLR in practice to continuously update SLR evidence in SE. Conclusion: The CSLR concept and process provide a feasible and systematic way to continuously incorporate new evidence into SLRs, supporting trustworthy and up-to-date evidence for SLRs in SE.},
  booktitle = {48th Euromicro Conference on Software Engineering and Advanced Applications, {SEAA} 2022, Gran Canaria, Spain, Aug 31st - Sept 2nd},
  pages     = {1-8},
  year      = {2022},
  urlAuthor_version = {http://www.inf.puc-rio.br/~kalinowski/publications/NapoleaoPHK22.pdf},  
  doi       = {10.1109/SEAA56994.2022.00078}
}

Context: New scientific evidence continuously arises with advances in Software Engineering (SE) research. Conventionally, Systematic Literature Reviews (SLRs) are not updated or updated intermittently, leaving gaps between updates, during which time the SLR may be missing crucial new evidence. Goal: We propose and evaluate a concept and process called Continuous Systematic Literature Review (CSLR) in SE. Method: To elaborate on the CSLR concept and process, we performed a synthesis of evidence by conducting a meta-ethnography, addressing knowledge from varied research areas. Furthermore, we conducted a case study to evaluate the CSLR process. Results: We describe the resulting CSLR process in BPMN format. The case study results provide indications on the importance and feasibility of applying CSLR in practice to continuously update SLR evidence in SE. Conclusion: The CSLR concept and process provide a feasible and systematic way to continuously incorporate new evidence into SLRs, supporting trustworthy and up-to-date evidence for SLRs in SE.

@inproceedings{VillamizarKL22b,
  author    = {Hugo Villamizar and Marcos Kalinowski and Helio Lopes},
  title     = {Towards Perspective-Based Specification of Machine Learning-Enabled Systems},
  abstract  = {Machine learning (ML) teams often work on a project just to realize the performance of the model is not good enough. Indeed, the success of ML-enabled systems involves aligning data with business problems, translating them into ML tasks, experimenting with algorithms, evaluating models, capturing data from users, among others. Literature has shown that ML-enabled systems are rarely built based on precise specifications for such concerns, leading ML teams to become misaligned due to incorrect assumptions, which may affect the quality of such systems and overall project success. In order to help addressing this issue, this paper describes our work towards a perspective-based approach for specifying ML-enabled systems. The approach involves analyzing a set of 45 ML concerns grouped into five perspectives: objectives, user experience, infrastructure, model, and data. The main contribution of this paper is to provide two new artifacts that can be used to help specifying ML-enabled systems: (i) the perspective-based ML task and concern diagram and (ii) the perspective-based ML specification template.},
  booktitle = {48th Euromicro Conference on Software Engineering and Advanced Applications, {SEAA} 2022, Gran Canaria, Spain, Aug 31st - Sept 2nd},
  pages     = {1-8},
  year      = {2022},
  urlAuthor_version = {http://www.inf.puc-rio.br/~kalinowski/publications/VillamizarKL22b.pdf},  
  doi       = {10.1109/SEAA56994.2022.00025}
}

Machine learning (ML) teams often work on a project just to realize the performance of the model is not good enough. Indeed, the success of ML-enabled systems involves aligning data with business problems, translating them into ML tasks, experimenting with algorithms, evaluating models, capturing data from users, among others. Literature has shown that ML-enabled systems are rarely built based on precise specifications for such concerns, leading ML teams to become misaligned due to incorrect assumptions, which may affect the quality of such systems and overall project success. In order to help addressing this issue, this paper describes our work towards a perspective-based approach for specifying ML-enabled systems. The approach involves analyzing a set of 45 ML concerns grouped into five perspectives: objectives, user experience, infrastructure, model, and data. The main contribution of this paper is to provide two new artifacts that can be used to help specifying ML-enabled systems: (i) the perspective-based ML task and concern diagram and (ii) the perspective-based ML specification template.

@inproceedings{VillamizarKL22,
  author = {Hugo Villamizar and Marcos Kalinowski and Helio Lopes},
  title = {A Catalogue of Concerns for Specifying Machine Learning-Enabled Systems},
  year = {2022},
  isbn = {},
  issn = {2675-0066},  
  publisher = {WERpapers},
  urlAuthor_version = {http://www.inf.puc-rio.br/~kalinowski/publications/VillamizarKL22.pdf},
  doi = {},
  abstract = {Requirements engineering (RE) activities for machine learning (ML) are not well-established and researched in the literature. Many issues and challenges exist when specifying, designing, and developing ML-enabled systems. Adding more focus on RE for ML can help to develop more reliable ML-enabled systems. Based on insights collected from previous work and industrial experiences, we propose a catalogue of 45 concerns to be considered when specifying ML-enabled systems, covering five different perspectives we identified as relevant for such systems: objectives, user experience, infrastructure, model, and data. Examples of such concerns include the execution engine and telemetry for the infrastructure perspective, and explainability and reproducibility for the model perspective. We conducted a focus group session with eight software professionals with experience developing ML-enabled systems to validate the importance, quality and feasibility of using our catalogue. The feedback allowed us to improve the catalogue and confirmed its practical relevance. The main research contribution of this work consists in providing a validated set of concerns grouped into perspectives that can be used by requirements engineers to support the specification of ML-enabled systems.},
  booktitle = {Workshop on Requirements Engineering, {WER} 2022, Natal, Brasil, Agosto 23-26},
  pages = {1-14},
  keywords = {Requirements engineering, requirements specification, perspectives, concerns, machine learning, artificial intelligence},
  location = {Natal, Brazil},
  series = {WER}
}

Requirements engineering (RE) activities for machine learning (ML) are not well-established and researched in the literature. Many issues and challenges exist when specifying, designing, and developing ML-enabled systems. Adding more focus on RE for ML can help to develop more reliable ML-enabled systems. Based on insights collected from previous work and industrial experiences, we propose a catalogue of 45 concerns to be considered when specifying ML-enabled systems, covering five different perspectives we identified as relevant for such systems: objectives, user experience, infrastructure, model, and data. Examples of such concerns include the execution engine and telemetry for the infrastructure perspective, and explainability and reproducibility for the model perspective. We conducted a focus group session with eight software professionals with experience developing ML-enabled systems to validate the importance, quality and feasibility of using our catalogue. The feedback allowed us to improve the catalogue and confirmed its practical relevance. The main research contribution of this work consists in providing a validated set of concerns grouped into perspectives that can be used by requirements engineers to support the specification of ML-enabled systems.

@inproceedings{GomesVEK22,
  author = {Gomes, Anna Luiza and Vianna, Get\'{u}lio and Escovedo, Tatiana and Kalinowski, Marcos},
  title = {Predicting IMDb Rating of TV Series with Deep Learning: The Case of Arrow},
  year = {2022},
  isbn = {9781450396981},
  publisher = {Association for Computing Machinery},
  urlAuthor_version = {http://www.inf.puc-rio.br/~kalinowski/publications/GomesVEK22.pdf},
  doi = {10.1145/3535511.3535520},
  abstract = {Context: The number of TV series offered nowadays is very high. Due to its large amount, many series are canceled due to a lack of originality that generates a low audience.Problem: Having a decision support system that can show why some shows are a huge success or not would facilitate the choices of renewing or starting a show.Solution: We studied the case of the series Arrow broadcasted by CW Network and used descriptive and predictive modeling techniques to predict the IMDb rating. We assumed that the theme of the episode would affect its evaluation by users, so the dataset is composed only by the director of the episode, the number of reviews that episode got, the percentual of each theme extracted by the Latent Dirichlet Allocation (LDA) model of an episode, the number of viewers from Wikipedia and the rating from IMDb. The LDA model is a generative probabilistic model of a collection of documents made up of words.IS Theory: This study was developed under the aegis of Computational Learning Theory, which aims to understand the fundamental principles of learning and contribute to designing better-automated learning methods applied to the entertainment business.Method: In this prescriptive research, the case study method was used, and its results were analyzed using a quantitative approach.Summary of Results: With the features of each episode, the model that performed the best to predict the rating was Catboost due to a similar mean squared error of the KNN model but a better standard deviation during the test phase. It was possible to predict IMDb ratings with an acceptable root mean squared error of 0.55.Contributions and Impact in the IS area: The results show that deep learning techniques can be applied to support decisions in the entertainment field, allowing facilitating the decisions of renewing or starting a show. The rationale for building the model is detailed throughout the paper and can be replicated for other contexts.},
  booktitle = {XVIII Brazilian Symposium on Information Systems},
  articleno = {9},
  pages = {1-6},
  keywords = {prediction, IMDb, Deep Learning, Machine Learning, tv-series, LDA},
  location = {Curitiba, Brazil},
  series = {SBSI}
}

Context: The number of TV series offered nowadays is very high. Due to its large amount, many series are canceled due to a lack of originality that generates a low audience.Problem: Having a decision support system that can show why some shows are a huge success or not would facilitate the choices of renewing or starting a show.Solution: We studied the case of the series Arrow broadcasted by CW Network and used descriptive and predictive modeling techniques to predict the IMDb rating. We assumed that the theme of the episode would affect its evaluation by users, so the dataset is composed only by the director of the episode, the number of reviews that episode got, the percentual of each theme extracted by the Latent Dirichlet Allocation (LDA) model of an episode, the number of viewers from Wikipedia and the rating from IMDb. The LDA model is a generative probabilistic model of a collection of documents made up of words.IS Theory: This study was developed under the aegis of Computational Learning Theory, which aims to understand the fundamental principles of learning and contribute to designing better-automated learning methods applied to the entertainment business.Method: In this prescriptive research, the case study method was used, and its results were analyzed using a quantitative approach.Summary of Results: With the features of each episode, the model that performed the best to predict the rating was Catboost due to a similar mean squared error of the KNN model but a better standard deviation during the test phase. It was possible to predict IMDb ratings with an acceptable root mean squared error of 0.55.Contributions and Impact in the IS area: The results show that deep learning techniques can be applied to support decisions in the entertainment field, allowing facilitating the decisions of renewing or starting a show. The rationale for building the model is detailed throughout the paper and can be replicated for other contexts.

@ARTICLE{LaignerEtAl21,
  author={Rodrigo Laigner and Yongluan Zhou and Marcos Antonio Vaz Salles and Yijian Liu and Marcos Kalinowski},
  journal={Proceedings of the VLDB Endowment}, 
  title={Data Management in Microservices: State of the Practice, Challenges, and Research Directions}, 
  year={2021},
  volume={14},
  number={13},
  pages={3348-3361},
  note={},  
  urlAuthor_version= {http://www.inf.puc-rio.br/~kalinowski/publications/LaignerEtAl21.pdf},
  doi={10.14778/3484224.3484232},
  keywords={Microservices, Data Management},
  abstract={Microservices have become a popular architectural style for data-driven applications, given their ability to functionally decompose an application into small and autonomous services to achieve scalability, strong isolation, and specialization of database systems to the workloads and data formats of each service. Despite the accelerating industrial adoption of this architectural style, an investigation of the state of the practice and challenges practitioners face regarding data management in microservices is lacking. To bridge this gap, we conducted a systematic literature review of representative articles reporting the adoption of microservices, we analyzed a set of popular open-source microservice applications, and we conducted an online survey to cross-validate the findings of the previous steps with the perceptions and experiences of over 120 experienced practitioners and researchers. Through this process, we were able to categorize the state of practice of data management in microservices and observe several foundational challenges that cannot be solved by software engineering practices alone, but rather require system-level support to alleviate the burden imposed on practitioners. We discuss the shortcomings of state-of-the-art database systems regarding microservices and we conclude by devising a set of features for microservice-oriented database systems.}
}

Microservices have become a popular architectural style for data-driven applications, given their ability to functionally decompose an application into small and autonomous services to achieve scalability, strong isolation, and specialization of database systems to the workloads and data formats of each service. Despite the accelerating industrial adoption of this architectural style, an investigation of the state of the practice and challenges practitioners face regarding data management in microservices is lacking. To bridge this gap, we conducted a systematic literature review of representative articles reporting the adoption of microservices, we analyzed a set of popular open-source microservice applications, and we conducted an online survey to cross-validate the findings of the previous steps with the perceptions and experiences of over 120 experienced practitioners and researchers. Through this process, we were able to categorize the state of practice of data management in microservices and observe several foundational challenges that cannot be solved by software engineering practices alone, but rather require system-level support to alleviate the burden imposed on practitioners. We discuss the shortcomings of state-of-the-art database systems regarding microservices and we conclude by devising a set of features for microservice-oriented database systems.

@inproceedings{FerreiraKGMLB21,
  author    = {Bruna Ferreira and Marcos Kalinowski and Marcus V.C. Gomes and Marcelo C. Marques and Helio Lopes and Simone D. J. Barbosa},
  title     = {Investigating Problem Definition and End-User Involvement in Agile Projects that Use Lean Inceptions},
  abstract  = {Agile methods focus on high quality, frequent deliveries, and customer and development team communication. Despite emphasizing customer involvement, in practice end users commonly do not participate in the early stages of these methods; instead, other business-related customers represent them in defining end-user needs. As a result, the produced software often does not satisfy the actual users’ needs, and the User Experience (UX) suffers. User-centered design, Design Thinking, and Lean Startup have been combined with agile methods to support defining the problem focused on user needs and producing effective software. Lean Inception (LI) represents a workshop that combines Design Thinking and Lean Startup. However, the effectiveness of problem definition and end-user participation in LIs is little explored. This paper presents a case study addressing this gap, investigating the use of LIs to define the problem and how the participation of end users in this context contributes to the resulting UX. We conducted ten semi-structured interviews with customers and end-users of an agile software project using LI. Our analysis suggests that, while LI supports the identification of business goals, more details about user needs and software requirements are needed to support designers and developers in understanding the problem and evaluating its feasibility. We also identified a mischaracterization of user journeys and personas in LIs due to the short time dedicated to their creation, hampering the discussion of relevant UX issues. Finally, we observed that, when end users were present in the early stages, they helped the development team to design software that fits into their routine, improving the UX and increasing its adoption. We qualitatively discuss these issues and gains of directly involving end users and not “proxy” stakeholders in LI’s.},
  booktitle = {Proceedings of the XX Brazilian Symposium on Software Quality, {SBQS'21}, Brazil, November 8-11},
  pages     = {1-10},
  note      = {},
  year      = {2021},
  urlAuthor_version      = {http://www.inf.puc-rio.br/~kalinowski/publications/FerreiraKGMLB21.pdf},
  doi       = {https://doi.org/10.1145/3493244.3493268},
}

Agile methods focus on high quality, frequent deliveries, and customer and development team communication. Despite emphasizing customer involvement, in practice end users commonly do not participate in the early stages of these methods; instead, other business-related customers represent them in defining end-user needs. As a result, the produced software often does not satisfy the actual users’ needs, and the User Experience (UX) suffers. User-centered design, Design Thinking, and Lean Startup have been combined with agile methods to support defining the problem focused on user needs and producing effective software. Lean Inception (LI) represents a workshop that combines Design Thinking and Lean Startup. However, the effectiveness of problem definition and end-user participation in LIs is little explored. This paper presents a case study addressing this gap, investigating the use of LIs to define the problem and how the participation of end users in this context contributes to the resulting UX. We conducted ten semi-structured interviews with customers and end-users of an agile software project using LI. Our analysis suggests that, while LI supports the identification of business goals, more details about user needs and software requirements are needed to support designers and developers in understanding the problem and evaluating its feasibility. We also identified a mischaracterization of user journeys and personas in LIs due to the short time dedicated to their creation, hampering the discussion of relevant UX issues. Finally, we observed that, when end users were present in the early stages, they helped the development team to design software that fits into their routine, improving the UX and increasing its adoption. We qualitatively discuss these issues and gains of directly involving end users and not “proxy” stakeholders in LI’s.

@inproceedings{SaraivaHCK21,
  author    = {Lorenzo Saraiva and Edward Hermann Haeusler and Vaston Costa and Marcos Kalinowski},
  title     = {Non-Functional Requirements for Quantum Programs},
  abstract  = {Quantum computing is moving from a purely theoretical area to an area with practical applications, allowing considerable performance efficiency improvements. The goal of this paper is to discuss non-functional requirements for quantum programs. Based on experiences developing quantum software for real quantum hardware we analyze hardware-related constraints and derive a set of generic non-functional requirements for this type of program. We identified a set of five performance efficiency and reliability related non-functional requirements that should considered when implementing a quantum program for a quantum device. We also discuss available solution options to address the requirements. There are high level solutions to deal with the hardware-related constraints described in our identified requirements. While many of the them are specific to quantum programming languages and technologies, the scientific community is engaging to integrate these kind of solutions into the quantum software engineering life cycle in an agnostic way regarding quantum programming languages and technologies.},
  booktitle = {2nd Quantum Software Engineering and Technology Workshop (QSET), co-located with IEEE International Conference on Quantum Computing and Engineering (QCE21), Virtual Conference, Oct 19},
  pages     = {},
  year      = {2021},
  urlAuthor_version      = {http://www.inf.puc-rio.br/~kalinowski/publications/SaraivaHCK21.pdf},
  doi       = {},
}

Quantum computing is moving from a purely theoretical area to an area with practical applications, allowing considerable performance efficiency improvements. The goal of this paper is to discuss non-functional requirements for quantum programs. Based on experiences developing quantum software for real quantum hardware we analyze hardware-related constraints and derive a set of generic non-functional requirements for this type of program. We identified a set of five performance efficiency and reliability related non-functional requirements that should considered when implementing a quantum program for a quantum device. We also discuss available solution options to address the requirements. There are high level solutions to deal with the hardware-related constraints described in our identified requirements. While many of the them are specific to quantum programming languages and technologies, the scientific community is engaging to integrate these kind of solutions into the quantum software engineering life cycle in an agnostic way regarding quantum programming languages and technologies.

@inproceedings{FurtadoOK21,
  author    = {Viviane Furtado and
               Edson OliveiraJr and
               Marcos Kalinowski},
  title     = {Guidelines for Promoting Software Product Line Experiments},
  abstract  = {The importance of experimentation for Software Engineering research has been notably established in the last years. The software engineering community has discussed how to proper report and evaluate experiments using different approaches, such as quality criteria, scales, and checklists. Nevertheless, there are no guidelines to support researchers and practitioners active in straightforward software engineering research areas, as in Software Product Lines (SPL), at conducting experiments. We hypothesize that experimentation guidelines may aid such a specific area by providing advice and actual excerpts reflecting good practices of SPL experimentation, thus experimentally evolving this area. Therefore, the goal of this paper is to provide guidelines for properly reporting and promoting SPL experiments. We defined such guidelines based on well-known software engineering experiment reports, quality evaluation checklists, and data extracted from 211 SPL experiments identified in a systematic mapping study. We evaluated the guidelines with a qualitative study with SPL and experimentation experts applying open and axial coding procedures. The evaluation enabled us to improve the guidelines. The resulting guidelines contain specific advice to researchers active in SPL and provide examples taken from published SPL experiments. The experts’ positive points indicate that the proposed guidelines can aid SPL researchers and practitioners. Sharing the resulting guidelines could support conducting SPL experiments and allow further area evolution based on prospective experiment replications and reproductions from well-designed and reported experiments.},
  booktitle = {Brazilian Symposium on Software Components, Architectures and Reuse, {SBCARS} 2021, Joinville, Brazil, September 27-28, 2021},
  pages     = {31-40},
  year      = {2021},
  urlAuthor_version      = {http://www.inf.puc-rio.br/~kalinowski/publications/FurtadoOK21.pdf},
  doi       = {10.1145/3483899.3483909},
}

The importance of experimentation for Software Engineering research has been notably established in the last years. The software engineering community has discussed how to proper report and evaluate experiments using different approaches, such as quality criteria, scales, and checklists. Nevertheless, there are no guidelines to support researchers and practitioners active in straightforward software engineering research areas, as in Software Product Lines (SPL), at conducting experiments. We hypothesize that experimentation guidelines may aid such a specific area by providing advice and actual excerpts reflecting good practices of SPL experimentation, thus experimentally evolving this area. Therefore, the goal of this paper is to provide guidelines for properly reporting and promoting SPL experiments. We defined such guidelines based on well-known software engineering experiment reports, quality evaluation checklists, and data extracted from 211 SPL experiments identified in a systematic mapping study. We evaluated the guidelines with a qualitative study with SPL and experimentation experts applying open and axial coding procedures. The evaluation enabled us to improve the guidelines. The resulting guidelines contain specific advice to researchers active in SPL and provide examples taken from published SPL experiments. The experts’ positive points indicate that the proposed guidelines can aid SPL researchers and practitioners. Sharing the resulting guidelines could support conducting SPL experiments and allow further area evolution based on prospective experiment replications and reproductions from well-designed and reported experiments.

@inproceedings{LaignerKalinowski21,
  author    = {Rodrigo Laigner and Marcos Kalinowski},
  title     = {Cataloging Dependency Injection Anti-Patterns in Software Systems},
  booktitle = {Concurso de Teses e Disserta{\c{c}}ões em Engenharia de Software, {SBES CTD-ES} 2021, Online, Sep 27-30},
  pages     = {1-2},
  note      = {<font color="red">Summary for the "Second Best Brazilian Software Engineering MSc Dissertation Award", received at SBES 2021. Student: Rodrigo Laigner, Advisor: Marcos Kalinowski</font>},  
  year      = {2021},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/LaignerKalinowski21.pdf},  
  doi       = {},
}

@inproceedings{AlonsoKVFB21,
  author    = {Silvio Alonso and Marcos Kalinowski and Marx Viana and Bruna Ferreira and Simone Barbosa},
  title     = {A Systematic Mapping Study on the Use of Software Engineering Practices to Develop MVPs},
  abstract  = {[Background] Many startup environments and even traditional software companies have embraced the use of MVPs (Minimum Viable Products) to allow quickly experimenting solution options. The MVP concept has influenced the way in which development teams apply Software Engineering~(SE) practices. However, the overall understanding of this influence of MVPs on SE practices is still poor.
    [Objective] Our goal is to characterize the publication landscape on practices that have been used in the context of software MVPs. 
    [Method] We conducted a systematic mapping study using a hybrid search strategy that consists of a database search and parallel forward and backward snowballing.
    [Results] We identified 33 papers, published between 2013 and 2020. We observed some trends related to MVP ideation and evaluation practices. For instance, regarding ideation, we found six different approaches (e.g., Design Thinking, Lean Inception) and mainly informal end-user involvement practices (e.g., workshops, interviews). For evaluation there is an emphasis on end-user validations based on practices such as usability tests, A/B testing, and usage data analysis. However, there is still limited research related to MVP technical feasibility assessment and effort estimation. We also observed a lack of scientific rigor in many of the identified studies.
    [Conclusion] Our analysis suggests that there are opportunities for solution proposals to address gaps concerning technical feasibility assessment and effort estimation. Also, more effort needs to be invested into empirically evaluating the existing MVP-related practices.},
  booktitle = {47th Euromicro Conference on Software Engineering and Advanced Applications, {SEAA} 2021, Palermo, Italy, Sep 1-3},
  pages     = {62-69},
  note      = {<font color="red">Best Paper Candidate, invited for journal extension.</font>},    
  year      = {2021},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/AlonsoKVFB21.pdf},    
  doi       = {10.1109/SEAA53835.2021.00017},
}

[Background] Many startup environments and even traditional software companies have embraced the use of MVPs (Minimum Viable Products) to allow quickly experimenting solution options. The MVP concept has influenced the way in which development teams apply Software Engineering (SE) practices. However, the overall understanding of this influence of MVPs on SE practices is still poor. [Objective] Our goal is to characterize the publication landscape on practices that have been used in the context of software MVPs. [Method] We conducted a systematic mapping study using a hybrid search strategy that consists of a database search and parallel forward and backward snowballing. [Results] We identified 33 papers, published between 2013 and 2020. We observed some trends related to MVP ideation and evaluation practices. For instance, regarding ideation, we found six different approaches (e.g., Design Thinking, Lean Inception) and mainly informal end-user involvement practices (e.g., workshops, interviews). For evaluation there is an emphasis on end-user validations based on practices such as usability tests, A/B testing, and usage data analysis. However, there is still limited research related to MVP technical feasibility assessment and effort estimation. We also observed a lack of scientific rigor in many of the identified studies. [Conclusion] Our analysis suggests that there are opportunities for solution proposals to address gaps concerning technical feasibility assessment and effort estimation. Also, more effort needs to be invested into empirically evaluating the existing MVP-related practices.

@inproceedings{AlmeidaKF21,
  author    = {Clauvin Almeida and Marcos Kalinowski and Bruno Feijó},
  title     = {A Systematic Mapping of Negative Effects of Gamification in Education/Learning Systems},
  abstract  = {While most research shows positive effects of gamification, the focus on its adverse effects is considerably smaller. Having this in mind, we conducted a systematic mapping study of the negative effects of game design elements on education/learning systems. The study revealed 77 papers reporting undesired effects of game design elements. We found that badges, competitions, leaderboards, and points are the game design elements most often reported as causing negative effects. The most cited negative effects concern lack of effect, lack of understanding, irrelevance, motivational issues, and worsened performance. The ethical issue of cheating was also often reported. As part of our results, we map the relations between game design elements and the negative effects that they may cause. Our mapping study can help gamification designers make more informed decisions when selecting game design elements to be included in education/learning systems, raising awareness on potential negative effects.},
  booktitle = {47th Euromicro Conference on Software Engineering and Advanced Applications, {SEAA} 2021, Palermo, Italy, Sep 1-3},
  pages     = {17-24},
  note      = {<font color="red">Best Paper Candidate, invited for journal extension.</font>},  
  year      = {2021},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/AlmeidaKF21.pdf},  
  doi       = {10.1109/SEAA53835.2021.00011},
}

While most research shows positive effects of gamification, the focus on its adverse effects is considerably smaller. Having this in mind, we conducted a systematic mapping study of the negative effects of game design elements on education/learning systems. The study revealed 77 papers reporting undesired effects of game design elements. We found that badges, competitions, leaderboards, and points are the game design elements most often reported as causing negative effects. The most cited negative effects concern lack of effect, lack of understanding, irrelevance, motivational issues, and worsened performance. The ethical issue of cheating was also often reported. As part of our results, we map the relations between game design elements and the negative effects that they may cause. Our mapping study can help gamification designers make more informed decisions when selecting game design elements to be included in education/learning systems, raising awareness on potential negative effects.

@inproceedings{VillamizarEK21,
  author    = {Hugo Villamizar and Tatiana Escovedo and Marcos Kalinowski},
  title     = {Requirements Engineering for Machine Learning: A Systematic Mapping Study},
  abstract  = {Machine learning (ML) has become a core feature for today's real-world applications, making it a trending topic for the software engineering community. Requirements Engineering (RE) is no stranger to this and its main conferences have included workshops aiming at discussing RE in the context of ML. However, current research on the intersection between RE and ML mainly focuses on using ML techniques to support RE activities rather than on exploring how RE can improve the development of ML-based systems. This paper concerns a systematic mapping study aiming at characterizing the publication landscape of RE for ML-based systems, outlining research contributions and  contemporary gaps for future research. In total, we identified 35 studies that met our inclusion criteria. We found several different types of contributions, in the form of analyses, approaches, checklists and guidelines, quality models, and taxonomies. We discuss gaps by mapping these contributions against the RE topics to which they were contributing and their type of empirical evaluation. We also identified quality characteristics that are particularly relevant for the ML context (e.g., data quality, explainability, fairness, safety, and transparency). Main reported challenges are related to the lack of validated RE techniques, the fragmented and incomplete understanding of NFRs for ML, and difficulties in handling customer expectations. There is a need for future research on the topic to reveal best practices and to propose and investigate approaches that are suitable to be used in practice.},
  booktitle = {47th Euromicro Conference on Software Engineering and Advanced Applications, {SEAA} 2021, Palermo, Italy, Sep 1-3},
  pages     = {29-36},
  year      = {2021},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/VillamizarEK21.pdf},      
  doi       = {10.1109/SEAA53835.2021.00013},
}

Machine learning (ML) has become a core feature for today's real-world applications, making it a trending topic for the software engineering community. Requirements Engineering (RE) is no stranger to this and its main conferences have included workshops aiming at discussing RE in the context of ML. However, current research on the intersection between RE and ML mainly focuses on using ML techniques to support RE activities rather than on exploring how RE can improve the development of ML-based systems. This paper concerns a systematic mapping study aiming at characterizing the publication landscape of RE for ML-based systems, outlining research contributions and contemporary gaps for future research. In total, we identified 35 studies that met our inclusion criteria. We found several different types of contributions, in the form of analyses, approaches, checklists and guidelines, quality models, and taxonomies. We discuss gaps by mapping these contributions against the RE topics to which they were contributing and their type of empirical evaluation. We also identified quality characteristics that are particularly relevant for the ML context (e.g., data quality, explainability, fairness, safety, and transparency). Main reported challenges are related to the lack of validated RE techniques, the fragmented and incomplete understanding of NFRs for ML, and difficulties in handling customer expectations. There is a need for future research on the topic to reveal best practices and to propose and investigate approaches that are suitable to be used in practice.

@inproceedings{TeixeiraFDBNVKSCL21,
  author    = {Bianca Teixeira and
               Bruna Ferreira and
               Andr{\'e} Damasceno and
               Simone Diniz Junqueira Barbosa and
               Cassia Novello and
               Hugo Villamizar and
               Marcos Kalinowski and
               Thuener Silva and
               Jacques Chueke and
               Helio Lopes and
               Andr{\'e} Kuramoto and
               Bruno Itagyba and
               Cristiane Salgado and
               Sidney Comandulli and
               Marinho Fischer and                                             
               Leonardo Fialho},
  title     = {Lessons Learned from a Lean R&D Project},
  abstract  = {In a partnership between academia and industry, we report our experience in applying the Lean R&D approach in the IAGO project, an oil and gas machine-learning-based dashboard. The approach is grounded in continuous experimentation through agile development, beginning with a Lean Inception workshop to define a Minimal Viable Product. Then, after technical feasibility assessments and conception phases, Scrum-based development begins, continually testing business hypotheses. We discuss our experiences in following the approach, and report developers' perceptions gathered through interviews and user evaluations of the final product. We found that the Lean Inception works well for aligning expectations and objectives among stakeholders, but it is not enough to level the domain knowledge among developers. The participation of end users in the workshop and throughout the project, as well as constant communication among all stakeholders, is very important to deliver appropriate solutions.},
  booktitle = {{ICEIS} 2021 - Proceedings of the 23rd International Conference on Enterprise Information Systems, Online Streaming, April 26-28,
               2021},
  pages     = {345-352},
  note      = {},        
  year      = {2021},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/TeixeiraFDBNVKSCL21.pdf},
  doi       = {10.5220/0010459803450352},
}

In a partnership between academia and industry, we report our experience in applying the Lean R&D approach in the IAGO project, an oil and gas machine-learning-based dashboard. The approach is grounded in continuous experimentation through agile development, beginning with a Lean Inception workshop to define a Minimal Viable Product. Then, after technical feasibility assessments and conception phases, Scrum-based development begins, continually testing business hypotheses. We discuss our experiences in following the approach, and report developers' perceptions gathered through interviews and user evaluations of the final product. We found that the Lean Inception works well for aligning expectations and objectives among stakeholders, but it is not enough to level the domain knowledge among developers. The participation of end users in the workshop and throughout the project, as well as constant communication among all stakeholders, is very important to deliver appropriate solutions.

@inproceedings{TorresDSSKISCVF21,
  author    = {Pedro Torres and
               Andre Davys and
               Thuener Silva and
               Luiz Schirmer and
               Andre Kuramoto and
               Bruno Itagyba and
               Cristiane Salgado and
               Sidney Comandulli and
               Patricia Ventura and
               Leonardo Fialho and
               Marinho Fischer and
               Marcos Kalinowski and
               Simone Barbosa and
               Helio Lopes},
  title     = {A Robust Real-time Component for Personal Protective Equipment Detection in an Industrial Setting},
  abstract  = {In large industries, such as construction, metallurgy, and oil, workers are continually exposed to various hazards in their workplace. Accordingly to the International Labor Organization (ILO), there are 340 million occupational accidents annually. Personal Protective Equipment (PPE) is used to ensure the essential protection of workers’ health and safety. There is a great effort to ensure that these types of equipment are used properly. In such an environment, it is common to have closed-circuit television (CCTV) cameras to monitor workers, as those can be used to verify the PPE’s proper usage. Some works address this problem using CCTV images; however, they frequently can not deal with multiples safe equipment usage detection and others even skip the verification phase, making only the detection. In this paper, we propose a novel cognitive safety analysis component for a monitoring system. This component acts to detect the proper usage of PPE’s in real-time using data stream from regular CCTV cameras. We built the system component based on the top of state-of-art deep learning techniques for object detection. The methodology is robust with consistent and promising results for Mean Average Precision (80.19% mAP) and can act in real-time (80 FPS).},
  booktitle = {{ICEIS} 2021 - Proceedings of the 23rd International Conference on Enterprise Information Systems, Online Streaming, April 26-28,
               2021},
  pages     = {693-700},
  note      = {},        
  year      = {2021},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/TorresDSSKISCVF21.pdf},
  doi       = {10.5220/0010452606930700},
}

In large industries, such as construction, metallurgy, and oil, workers are continually exposed to various hazards in their workplace. Accordingly to the International Labor Organization (ILO), there are 340 million occupational accidents annually. Personal Protective Equipment (PPE) is used to ensure the essential protection of workers’ health and safety. There is a great effort to ensure that these types of equipment are used properly. In such an environment, it is common to have closed-circuit television (CCTV) cameras to monitor workers, as those can be used to verify the PPE’s proper usage. Some works address this problem using CCTV images; however, they frequently can not deal with multiples safe equipment usage detection and others even skip the verification phase, making only the detection. In this paper, we propose a novel cognitive safety analysis component for a monitoring system. This component acts to detect the proper usage of PPE’s in real-time using data stream from regular CCTV cameras. We built the system component based on the top of state-of-art deep learning techniques for object detection. The methodology is robust with consistent and promising results for Mean Average Precision (80.19% mAP) and can act in real-time (80 FPS).

@inproceedings{WagnerEtAl21,
  author    = {Stefan Wagner and
               Daniel M{\'{e}}ndez Fern{\'{a}}ndez and
               Michael Felderer and
               Antonio Vetr{\`{o}} and
               Marcos Kalinowski and
               Roel J. Wieringa and
               Dietmar Pfahl and
               Tayana Conte and
               Marie{-}Therese Christiansson and
               Desmond Greer and
               Casper Lassenius and
               Tomi M{\"{a}}nnist{\"{o}} and
               Maleknaz Nayebi and
               Markku Oivo and
               Birgit Penzenstadler and
               Rafael Prikladnicki and
               Guenther Ruhe and
               Andr{\'{e}} Schekelmann and
               Sagar Sen and
               Rodrigo O. Sp{\'{\i}}nola and
               Ahmet Tuzcu and
               Jose Luis de la Vara and
               Dietmar Winkler},
  editor    = {Anne Koziolek and
               Ina Schaefer and
               Christoph Seidl},
  title     = {Status Quo in Requirements Engineering: {A} Theory and a Global Family
               of Surveys},
  booktitle = {Software Engineering 2021, Fachtagung des GI-Fachbereichs Softwaretechnik,
               22.-26. Februar 2021, Braunschweig/Virtuell},
  series    = {{LNI}},
  volume    = {{P-310}},
  pages     = {115--116},
  publisher = {Gesellschaft f{\"{u}}r Informatik e.V.},
  year      = {2021},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/WagnerEtAl21.pdf},
  doi       = {10.18420/SE2021\_45},
}

@article{VillamizarKGM20,
	author = {Hugo Villamizar and Marcos Kalinowski and Alessandro Garcia and Daniel Mendez},
	title = {An Efficient Approach for Reviewing Security-Related Aspects in Agile Requirements Specifications of Web Applications},
	journal = {Requirements Engineering},
	volume = {25},
	number = {4},
  note = {},	
  year = {2020},
	keywords = {},
	abstract = {Defects in requirements specifications can have severe consequences during the software development lifecycle. Some of them may result in poor product quality and/or time and budget overruns due to incorrect or missing quality characteristics, such as security. This characteristic requires special attention in web applications because they have become a target for manipulating sensible data. Several concerns make security difficult to deal with. For instance, security requirements are often misunderstood and improperly specified due to lack of security expertise and emphasis on security during early stages of software development. This often leads to unspecified or ill-defined security-related aspects. These concerns become even more challenging in agile contexts, where lightweight documentation is typically produced. To tackle this problem, we designed an approach for reviewing security-related aspects in agile requirements specifications of web applications. Our proposal considers user stories and security specifications as inputs and relates those user stories to security properties via Natural Language Processing. Based on the related security properties, our approach identifies high-level security requirements from the Open Web Application Security Project (OWASP) to be verified, and generates a reading technique to support reviewers in detecting defects. We evaluate our approach via three experiment trials conducted with 56 novice software engineers, measuring effectiveness, efficiency, usefulness, and ease of use. We compare our approach against using: (1) the OWASP high-level security requirements, and (2) a perspective-based approach as proposed in contemporary state of the art. The results strengthen our confidence that using our approach has a positive impact (with large effect size) on the performance of inspectors in terms of effectiveness and efficiency.},
	issn = {1432-010X},	
	pages = {439-468},	
	doi = {https://doi.org/10.1007/s00766-020-00338-w},
	urlAuthor_version= {http://www.inf.puc-rio.br/~kalinowski/publications/VillamizarKGM20.pdf},
}

Defects in requirements specifications can have severe consequences during the software development lifecycle. Some of them may result in poor product quality and/or time and budget overruns due to incorrect or missing quality characteristics, such as security. This characteristic requires special attention in web applications because they have become a target for manipulating sensible data. Several concerns make security difficult to deal with. For instance, security requirements are often misunderstood and improperly specified due to lack of security expertise and emphasis on security during early stages of software development. This often leads to unspecified or ill-defined security-related aspects. These concerns become even more challenging in agile contexts, where lightweight documentation is typically produced. To tackle this problem, we designed an approach for reviewing security-related aspects in agile requirements specifications of web applications. Our proposal considers user stories and security specifications as inputs and relates those user stories to security properties via Natural Language Processing. Based on the related security properties, our approach identifies high-level security requirements from the Open Web Application Security Project (OWASP) to be verified, and generates a reading technique to support reviewers in detecting defects. We evaluate our approach via three experiment trials conducted with 56 novice software engineers, measuring effectiveness, efficiency, usefulness, and ease of use. We compare our approach against using: (1) the OWASP high-level security requirements, and (2) a perspective-based approach as proposed in contemporary state of the art. The results strengthen our confidence that using our approach has a positive impact (with large effect size) on the performance of inspectors in terms of effectiveness and efficiency.

@article{RalphBATKKNYDT20,
	author = {Paul Ralph and Sebastian Baltes and Gianisa Adisaputri and Richard Torkar and Vladimir Kovalenko and Marcos Kalinowski and Nicole Novielli and Shin Yoo and Xavier Devroey and Xin Tan and Minghui Zhou and Burak Turhan and Rashina Hoda and Hideaki Hata and Gregorio Robles and Amin Milani Fard and Rana Alkadhi},
	title = {Pandemic Programming How COVID-19 affects software developers and how their organizations can help},
	journal = {Empirical Software Engineering},
	volume = {25},
	number = {},
  note = {},	
  year = {2020},
	keywords = {},
	abstract = {Background. As a novel coronavirus swept the world in early 2020, thousands of software developers began working from home. Many did so on short notice , under difficult and stressful conditions. Aim. This paper seeks to understand the effects of the pandemic on developers' wellbeing and productivity. Method. A questionnaire survey was created mainly from existing, validated scales. The questionnaire ran in 12 languages, with region-specific advertising strategies. The data was analyzed using non-parametric inferential statistics and structural equation modeling. Results. The questionnaire received 2225 usable responses from 53 countries. Factor analysis supported the validity of the scales and the structural model achieved a good fit (χ 2 fit = 10.8, CFI = 0.961, RMSEA = 0.051, SRMR = 0.067). Findings include: (1) developers' wellbeing P. Ralph, Dalhousie University, 2 Paul Ralph et al. and productivity are suffering; (2) productivity and wellbeing are closely related ; (3) disaster preparedness, fear related to the pandemic and home office ergonomics all affect wellbeing or productivity; (4) women, parents and people with disabilities may be disproportionately affected. Conclusion. To improve employee productivity, software companies should focus on maximizing employee wellbeing and improving the ergonomics of employees' home offices. Women, parents and disabled persons may require extra support.},
	issn = {1573-7616},	
	pages = {4927–4961},	
	doi = {https://doi.org/10.1007/s10664-020-09875-y},
	urlAuthor_version= {http://www.inf.puc-rio.br/~kalinowski/publications/RalphBATKKNYDT20.pdf},
}

Background. As a novel coronavirus swept the world in early 2020, thousands of software developers began working from home. Many did so on short notice , under difficult and stressful conditions. Aim. This paper seeks to understand the effects of the pandemic on developers' wellbeing and productivity. Method. A questionnaire survey was created mainly from existing, validated scales. The questionnaire ran in 12 languages, with region-specific advertising strategies. The data was analyzed using non-parametric inferential statistics and structural equation modeling. Results. The questionnaire received 2225 usable responses from 53 countries. Factor analysis supported the validity of the scales and the structural model achieved a good fit (χ 2 fit = 10.8, CFI = 0.961, RMSEA = 0.051, SRMR = 0.067). Findings include: (1) developers' wellbeing P. Ralph, Dalhousie University, 2 Paul Ralph et al. and productivity are suffering; (2) productivity and wellbeing are closely related ; (3) disaster preparedness, fear related to the pandemic and home office ergonomics all affect wellbeing or productivity; (4) women, parents and people with disabilities may be disproportionately affected. Conclusion. To improve employee productivity, software companies should focus on maximizing employee wellbeing and improving the ergonomics of employees' home offices. Women, parents and disabled persons may require extra support.

@article{OliveiraKKMB20,
	author = {Alessandreia Oliveira and Troy Kohwalter and Marcos Kalinowski and Leonardo Murta and Vanessa Braganholo},
	title = {XChange: A semantic diff approach for XML documents},
	journal = {Information Systems},
	volume = {94},
	number = {},
	note = {},      
	year = {2020},
	keywords = {},
	abstract = {XML documents are extensively used in several applications and evolve over time. Identifying the semantics of these changes becomes a fundamental process to understand their evolution. Existing approaches related to understanding changes (diff) in XML documents focus only on syntactic changes. These approaches compare XML documents based on their structure, without considering the associated semantics. However, for large XML documents, which have undergone many changes from a version to the next, a large number of syntactic changes in the document may correspond to fewer semantic changes, which are then easier to analyze and understand. For instance, increasing the annual salary and the gross pay, and changing the job title of an employee (three syntactic changes) may mean that this employee was promoted (one semantic change). In this paper, we explore this idea and present the XChange approach. XChange considers the semantics of the changes to calculate the diff of different versions of XML documents. For such, our approach analyzes the granular syntactic changes in XML attributes and elements using inference rules to combine them into semantic changes. Thus, differently from existing approaches, XChange proposes the use of syntactic changes in versions of an XML document to infer the real reason for the change and support the process of semantic diff. Results of an experimental study indicate that XChange can provide higher effectiveness and efficiency when used to understand changes between versions of XML documents when compared with the (syntactic) state-of-the-art approaches.},
	issn = {0306-4379},	
	pages = {101610:1-18},	
	doi = {10.1016/j.is.2020.101610},
	urlAuthor_version= {http://www.inf.puc-rio.br/~kalinowski/publications/OliveiraKKMB20.pdf},
}

XML documents are extensively used in several applications and evolve over time. Identifying the semantics of these changes becomes a fundamental process to understand their evolution. Existing approaches related to understanding changes (diff) in XML documents focus only on syntactic changes. These approaches compare XML documents based on their structure, without considering the associated semantics. However, for large XML documents, which have undergone many changes from a version to the next, a large number of syntactic changes in the document may correspond to fewer semantic changes, which are then easier to analyze and understand. For instance, increasing the annual salary and the gross pay, and changing the job title of an employee (three syntactic changes) may mean that this employee was promoted (one semantic change). In this paper, we explore this idea and present the XChange approach. XChange considers the semantics of the changes to calculate the diff of different versions of XML documents. For such, our approach analyzes the granular syntactic changes in XML attributes and elements using inference rules to combine them into semantic changes. Thus, differently from existing approaches, XChange proposes the use of syntactic changes in versions of an XML document to infer the real reason for the change and support the process of semantic diff. Results of an experimental study indicate that XChange can provide higher effectiveness and efficiency when used to understand changes between versions of XML documents when compared with the (syntactic) state-of-the-art approaches.

@article{WohlinMFK20,
	author = {Claes Wohlin and Emilia Mendes and Katia Felizardo and Marcos Kalinowski},
	title = {Guidelines for the search strategy to update systematic literature reviews in software engineering},
	journal = {Information and Software Technology},
	volume = {127},
	number = {},
	note = {},      
	year = {2020},
	keywords = {},
	abstract = {Context: Systematic Literature Reviews (SLRs) have been adopted within Software Engineering (SE) for more than a decade to provide meaningful summaries of evidence on several topics. Many of these SLRs are now potentially not fully up-to-date, and there are no standard proposals on how to update SLRs in SE.  Objective: The objective of this paper is to propose guidelines on how to best search for evidence when updating SLRs in SE, and to evaluate these guidelines using an SLR that was not employed during the formulation of the guidelines. Method: To propose our guidelines, we compare and discuss outcomes from applying different search strategies to identify primary studies in a published SLR, an SLR update, and two replications in the area of effort estimation. These guidelines are then evaluated using an SLR in the area of software ecosystems, its update and a replication. Results: The use of a single iteration forward snowballing with Google Scholar, and employing as a seed set the original SLR and its primary studies is the most cost-effective way to search for new evidence when updating SLRs. Furthermore, the importance of having more than one researcher involved in the selection of papers when applying the inclusion and exclusion criteria is highlighted through the results. Conclusions: Our proposed guidelines formulated based upon an effort estimation SLR, its update and two replications, were supported when using an SLR in the area of software ecosystems, its update and a replication. Therefore, we put forward that our guidelines ought to be adopted for updating SLRs in SE.},
	issn = {0950-5849},	
	pages = {106366:1-12},	
	doi = {10.1016/j.infsof.2020.106366},
	urlAuthor_version= {http://www.inf.puc-rio.br/~kalinowski/publications/WohlinMFK20.pdf},
}

Context: Systematic Literature Reviews (SLRs) have been adopted within Software Engineering (SE) for more than a decade to provide meaningful summaries of evidence on several topics. Many of these SLRs are now potentially not fully up-to-date, and there are no standard proposals on how to update SLRs in SE. Objective: The objective of this paper is to propose guidelines on how to best search for evidence when updating SLRs in SE, and to evaluate these guidelines using an SLR that was not employed during the formulation of the guidelines. Method: To propose our guidelines, we compare and discuss outcomes from applying different search strategies to identify primary studies in a published SLR, an SLR update, and two replications in the area of effort estimation. These guidelines are then evaluated using an SLR in the area of software ecosystems, its update and a replication. Results: The use of a single iteration forward snowballing with Google Scholar, and employing as a seed set the original SLR and its primary studies is the most cost-effective way to search for new evidence when updating SLRs. Furthermore, the importance of having more than one researcher involved in the selection of papers when applying the inclusion and exclusion criteria is highlighted through the results. Conclusions: Our proposed guidelines formulated based upon an effort estimation SLR, its update and two replications, were supported when using an SLR in the area of software ecosystems, its update and a replication. Therefore, we put forward that our guidelines ought to be adopted for updating SLRs in SE.

@article{MendesWFK20,
	author = {Emilia Mendes and Claes Wohlin and Katia Felizardo and Marcos Kalinowski},
	title = {When to update Systematic Literature Reviews in Software Engineering},
	journal = {Journal of Systems and Software},
	volume = {167},
	number = {},
	note = {},      
	year = {2020},
	keywords = {},
	abstract = {[Context] Systematic Literature Reviews (SLRs) have been adopted by the Software Engineering (SE) community for approximately 15 years to provide meaningful summaries of evidence on several topics. Many of these SLRs are now potentially outdated, and there are no systematic proposals on when to update SLRs in SE. [Objective] The goal of this paper is to provide recommendations on when to update SLRs in SE. [Method] We evaluated, using a three-step approach, a third-party decision framework (3PDF) employed in other fields, to decide whether SLRs need updating. First, we conducted a literature review of SLR updates in SE and contacted the authors to obtain their feedback relating to the usefulness of the 3PDF within the context of SLR updates in SE. Second, we used these authors’ feedback to see whether the framework needed any adaptation; none was suggested. Third, we applied the 3PDF to the SLR updates identified in our literature review. [Results] The 3PDF showed that 14 of the 20 SLRs did not need updating. This supports the use of a decision support mechanism (such as the 3PDF) to help the SE community decide when to update SLRs. [Conclusions] We put forward that the 3PDF should be adopted by the SE community to keep relevant evidence up to date and to avoid wasting effort with unnecessary updates.},
	issn = {0164-1212},	
	pages = {110607:1-24},	
	doi = {10.1016/j.jss.2020.110607},
	urlAuthor_version = {http://www.inf.puc-rio.br/~kalinowski/publications/MendesWFK20.pdf},
}

[Context] Systematic Literature Reviews (SLRs) have been adopted by the Software Engineering (SE) community for approximately 15 years to provide meaningful summaries of evidence on several topics. Many of these SLRs are now potentially outdated, and there are no systematic proposals on when to update SLRs in SE. [Objective] The goal of this paper is to provide recommendations on when to update SLRs in SE. [Method] We evaluated, using a three-step approach, a third-party decision framework (3PDF) employed in other fields, to decide whether SLRs need updating. First, we conducted a literature review of SLR updates in SE and contacted the authors to obtain their feedback relating to the usefulness of the 3PDF within the context of SLR updates in SE. Second, we used these authors’ feedback to see whether the framework needed any adaptation; none was suggested. Third, we applied the 3PDF to the SLR updates identified in our literature review. [Results] The 3PDF showed that 14 of the 20 SLRs did not need updating. This supports the use of a decision support mechanism (such as the 3PDF) to help the SE community decide when to update SLRs. [Conclusions] We put forward that the 3PDF should be adopted by the SE community to keep relevant evidence up to date and to avoid wasting effort with unnecessary updates.

@article{MouraoPMKMW20,
	author = {Erica Mourao and Joao Felipe Pimentel and Leonardo Murta and Marcos Kalinowski and Emilia Mendes and Claes Wohlin},
	title = {On the Performance of Hybrid Search Strategies for Systematic Literature Reviews in Software Engineering},
	journal = {Information and Software Technology},
	volume = {123},
	number = {},
	note = {},      
	year = {2020},
	keywords = {},
	abstract = {Context: When conducting a Systematic Literature Review (SLR), researchers usually face the challenge of designing a search strategy that appropriately balances result quality and review effort. Using digital library (or database) searches or snowballing alone may not be enough to achieve high-quality results. On the other hand, using both digital library searches and snowballing together may increase the overall review effort.
				Objective: The goal of this research is to propose and evaluate hybrid search strategies that selectively combine database searches with snowballing.
				Method: We propose four hybrid search strategies combining database searches in digital libraries with iterative, parallel, or sequential backward and forward snowballing. We simulated the strategies over three existing SLRs in SE that adopted both database searches and snowballing. We compared the outcome of digital library searches, snowballing, and hybrid strategies using precision, recall, and F-measure to investigate the performance of each strategy.
				Results: Our results show that, for the analyzed SLRs, combining database searches from the Scopus digital library with parallel or sequential snowballing achieved the most appropriate balance of precision and recall.
				Conclusion: We put forward that, depending on the goals of the SLR and the available resources, using a hybrid search strategy involving a representative digital library and parallel or sequential snowballing tends to represent an appropriate alternative to be used when searching for evidence in SLRs.},
	issn = {0950-5849},	
	pages = {106294:1-12},	
	doi = {10.1016/j.infsof.2020.106294},
	urlAuthor_version = {http://www.inf.puc-rio.br/~kalinowski/publications/MouraoPMKMW20.pdf},
}

Context: When conducting a Systematic Literature Review (SLR), researchers usually face the challenge of designing a search strategy that appropriately balances result quality and review effort. Using digital library (or database) searches or snowballing alone may not be enough to achieve high-quality results. On the other hand, using both digital library searches and snowballing together may increase the overall review effort. Objective: The goal of this research is to propose and evaluate hybrid search strategies that selectively combine database searches with snowballing. Method: We propose four hybrid search strategies combining database searches in digital libraries with iterative, parallel, or sequential backward and forward snowballing. We simulated the strategies over three existing SLRs in SE that adopted both database searches and snowballing. We compared the outcome of digital library searches, snowballing, and hybrid strategies using precision, recall, and F-measure to investigate the performance of each strategy. Results: Our results show that, for the analyzed SLRs, combining database searches from the Scopus digital library with parallel or sequential snowballing achieved the most appropriate balance of precision and recall. Conclusion: We put forward that, depending on the goals of the SLR and the available resources, using a hybrid search strategy involving a representative digital library and parallel or sequential snowballing tends to represent an appropriate alternative to be used when searching for evidence in SLRs.

@article{FariasMKS20,
	author = {M{\'{a}}rio Andr{\'{e}} de Freitas Farias and Manoel Gomes de Mendon{\c{c}}a and Marcos Kalinowski and Rodrigo O. Sp{\'{\i}}nola},
	title = {Identifying Self-Admitted Technical Debt through Code Comment Analysis with a Contextualized Vocabulary},
	journal = {Information and Software Technology},
	volume = {121},
	number = {},
	note = {},      
	year = {2020},
	keywords = {},
	abstract = {Context: Previous work has shown that we can use code comments analysis to detect Self-Admitted Technical Debt (SATD). However, current SATD identification strategies still return a large number of candidate SATD items, making the identification process laborious. Besides, those strategies do not allow the automatic identification of the type of debt of the SATD items. 
				Objective: This work intends to evaluate, improve, and apply a set of contextualized patterns we built to detect SATD using code comment analysis. We refer to this set of patterns as the TD identification vocabulary. 
				Method: We carry out three empirical studies. Firstly, 23 participants analyzed the patterns of the vocabulary and registered their level of importance to identify SATD items. In the second study, we performed a qualitative analysis to investigate the relation between each pattern and the types of TD. These two studies resulted in an improved vocabulary. Finally, we performed a feasibility study using the improved vocabulary, created based on the results of empirical studies I and II, considering three open source projects: ArgoUML, jEdit, and Lucene. We used the improved vocabulary to automatically identify SATD items and the types of debt that exist in those projects.   
				Results: The result was an improved vocabulary considering the level of importance of each pattern and the relationship between patterns and TD types to support the identification and classification of SATD items. More than half of the patterns were considered decisive or very decisive to detect SATD. Besides, using the improved vocabulary, we were able to find different TD types such as code, design, defect, documentation, and requirement debt.
				Conclusion: The studies allowed us to improve the vocabulary to identify SATD through code comments analysis. The results show that the use of pattern-based code comment analysis can contribute to improve existing methods, or create new ones, for identifying and classifying SATD items.},
	issn = {0950-5849},	
	pages = {106270:1-18},	
	doi = {10.1016/j.infsof.2020.106270},
	urlAuthor_version = {http://www.inf.puc-rio.br/~kalinowski/publications/FariasMKS20.pdf},
}

Context: Previous work has shown that we can use code comments analysis to detect Self-Admitted Technical Debt (SATD). However, current SATD identification strategies still return a large number of candidate SATD items, making the identification process laborious. Besides, those strategies do not allow the automatic identification of the type of debt of the SATD items. Objective: This work intends to evaluate, improve, and apply a set of contextualized patterns we built to detect SATD using code comment analysis. We refer to this set of patterns as the TD identification vocabulary. Method: We carry out three empirical studies. Firstly, 23 participants analyzed the patterns of the vocabulary and registered their level of importance to identify SATD items. In the second study, we performed a qualitative analysis to investigate the relation between each pattern and the types of TD. These two studies resulted in an improved vocabulary. Finally, we performed a feasibility study using the improved vocabulary, created based on the results of empirical studies I and II, considering three open source projects: ArgoUML, jEdit, and Lucene. We used the improved vocabulary to automatically identify SATD items and the types of debt that exist in those projects. Results: The result was an improved vocabulary considering the level of importance of each pattern and the relationship between patterns and TD types to support the identification and classification of SATD items. More than half of the patterns were considered decisive or very decisive to detect SATD. Besides, using the improved vocabulary, we were able to find different TD types such as code, design, defect, documentation, and requirement debt. Conclusion: The studies allowed us to improve the vocabulary to identify SATD through code comments analysis. The results show that the use of pattern-based code comment analysis can contribute to improve existing methods, or create new ones, for identifying and classifying SATD items.

@article{LageTK20,
	author = {Luiz Carlos da Fonseca Lage and Daniela Trevisan and Marcos Kalinowski},
	title = {Dívida T{\'e}cnica de Usabilidade em Projetos de Software: Um Estudo de Casos Múltiplos},
	journal = {iSys - Revista Brasileira de Sistemas de Informa{\c{c}}{\~a}o},
	volume = {},
	number = {},
	note = {},      
	year = {2020},
	keywords = {},
	abstract = {Background: Over the years, several studies were conducted aiming at understanding the Technical Debt (TD) phenomenon and its implications on software development. Most of these studies focus on source code related TD types. The absence of empirical studies on usability debt motivated our research. Aims: The goal of this paper is to provide an initial usability debt characterization in software projects regarding its occurrence, type, and resolution effort. Method: We conducted a multi-case study, analyzing TD items of five software projects from four Brazilian public companies. Results: After several steps of selection, classification, and validation, we identified 145 TD items in the change management systems used in the projects.  The analysis of these items allowed us to observe that 13.8% of the TD items concerned usability debt (ranging from10.4% to 20.8% in the five projects). The identified usability debt items cover a range of relevant usability issues, violating eight out of the ten Nielsen usability heuristics. Regarding effort for paying the TD, measured in man hours estimated by the project managers for resolving the TD items, usability debt items require a relatively low effort, ranging from 5.1% to 6.7% of the total TD resolution effort in the analyzed projects. Conclusions: Considering that usability TD items are frequent, concern relevant usability issues and require low effort for their payment, we put forward that actions for identifying and paying this type of TD should receive high priority in TD management strategies.},
	issn = {1984-2902},	
	pages = {1-26},	
	doi = {},
	urlAuthor_version = {http://www.inf.puc-rio.br/~kalinowski/publications/LageTK20.pdf},
}

Background: Over the years, several studies were conducted aiming at understanding the Technical Debt (TD) phenomenon and its implications on software development. Most of these studies focus on source code related TD types. The absence of empirical studies on usability debt motivated our research. Aims: The goal of this paper is to provide an initial usability debt characterization in software projects regarding its occurrence, type, and resolution effort. Method: We conducted a multi-case study, analyzing TD items of five software projects from four Brazilian public companies. Results: After several steps of selection, classification, and validation, we identified 145 TD items in the change management systems used in the projects. The analysis of these items allowed us to observe that 13.8% of the TD items concerned usability debt (ranging from10.4% to 20.8% in the five projects). The identified usability debt items cover a range of relevant usability issues, violating eight out of the ten Nielsen usability heuristics. Regarding effort for paying the TD, measured in man hours estimated by the project managers for resolving the TD items, usability debt items require a relatively low effort, ranging from 5.1% to 6.7% of the total TD resolution effort in the analyzed projects. Conclusions: Considering that usability TD items are frequent, concern relevant usability issues and require low effort for their payment, we put forward that actions for identifying and paying this type of TD should receive high priority in TD management strategies.

@inproceedings{CorreiaEtAl20,
  author    = {Jo{\~a}o Lucas Correia and Juliana Alves Pereira and Rafael de Mello and Alessandro Garcia and Baldoino Fonseca and Marcio Ribeiro and Rohit Gheyi and Willy Tiengo and Marcos Kalinowski and Renato Cerqueira},
  title     = {Brazilian Data Scientists: Revealing their Challenges and Practices on Machine Learning Model Development},
  abstract  = {Data scientists often develop machine learning models to solve a variety of problems in the industry and academy. To build these models, these professionals usually perform activities that are also performed in the traditional software development lifecycle, such as eliciting and implementing requirements. One might argue that data scientists could rely on the engineering of traditional software development to build machine learning models. However, machine learning development presents certain characteristics, which may raise challenges that lead to the need for adopting new practices. The literature lacks in characterizing this knowledge from the perspective of the data scientists. In this paper, we characterize challenges and practices addressing the engineering of machine learning models that deserve attention from the research community. To this end, we performed a qualitative study with eight data scientists across five different companies having different levels of experience in developing machine learning models. Our findings suggest that: (i) data processing and feature engineering are the most challenging stages in the development of machine learning models; (ii) it is essential synergy between data scientists and domain experts in most of stages; and (iii) the development of machine learning models lacks the support of a well-engineered process.},
  booktitle = {Proceedings of the XIX Brazilian Symposium on Software Quality, {SBQS'20}, Brazil, December 1st - December 4th},
  pages     = {1-10},
  note      = {},
  year      = {2020},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/CorreiaEtAl20.pdf},
  doi       = {},
}

Data scientists often develop machine learning models to solve a variety of problems in the industry and academy. To build these models, these professionals usually perform activities that are also performed in the traditional software development lifecycle, such as eliciting and implementing requirements. One might argue that data scientists could rely on the engineering of traditional software development to build machine learning models. However, machine learning development presents certain characteristics, which may raise challenges that lead to the need for adopting new practices. The literature lacks in characterizing this knowledge from the perspective of the data scientists. In this paper, we characterize challenges and practices addressing the engineering of machine learning models that deserve attention from the research community. To this end, we performed a qualitative study with eight data scientists across five different companies having different levels of experience in developing machine learning models. Our findings suggest that: (i) data processing and feature engineering are the most challenging stages in the development of machine learning models; (ii) it is essential synergy between data scientists and domain experts in most of stages; and (iii) the development of machine learning models lacks the support of a well-engineered process.

@inproceedings{MendoncaK20,
  author    = {Diogo S. Mendon\c{c}a and Marcos Kalinowski},
  title     = {Towards Pratical Reuse of Custom Static Analysis Rules for Defect Localization},
  abstract  = {[Context] Several static analysis tools allow the development of custom rules for locating application-specific defects. Although this feature is powerful and commonly available, it is not well explored in practice. Custom static analysis rules can check design and policies that are shared between applications, allowing the reuse of rules. However, the benefits, scope, and concerns that software engineers should have on reusing custom static analysis rules are unknown. [Goal] In this preliminary study, we investigate the reuse of custom static analysis rules produced by applying Pattern-Driven Maintenance (PDM). PDM is a method to locate defect patterns in web applications that produces custom static analysis rules as output. [Method] We selected a set of rules produced by a previous usage of the PDM method and applied them to other three applications in two contexts, within the same company where the rules were produced, and in other companies. [Results] We successfully reused some rules in both scenarios with minor adjustments, finding new defects to be fixed. The reuse of rules could discard from 58-90% of source code locations found by a naive search for the defects, reducing verification effort. However, the reused rules need adjustments to improve precision for defect localization, as precision ranged from 40-75%. Finally, we identified factors that have an impact on reusing custom rules. [Conclusions] We put forward that reusing customized static analysis rules can be beneficial, in particular when similarities in the architecture and programming style are observed. However, adjustment of the rules might be needed to enable effective reuse. We shared our insights and methodology on how to reuse custom static analysis rules properly.},
  booktitle = {Proceedings of the XIX Brazilian Symposium on Software Quality, {SBQS'20}, Brazil, December 1st - December 4th},
  pages     = {24:1-24:10},
  note      = {},
  year      = {2020},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/MendoncaK20.pdf},
  doi       = {},
}

[Context] Several static analysis tools allow the development of custom rules for locating application-specific defects. Although this feature is powerful and commonly available, it is not well explored in practice. Custom static analysis rules can check design and policies that are shared between applications, allowing the reuse of rules. However, the benefits, scope, and concerns that software engineers should have on reusing custom static analysis rules are unknown. [Goal] In this preliminary study, we investigate the reuse of custom static analysis rules produced by applying Pattern-Driven Maintenance (PDM). PDM is a method to locate defect patterns in web applications that produces custom static analysis rules as output. [Method] We selected a set of rules produced by a previous usage of the PDM method and applied them to other three applications in two contexts, within the same company where the rules were produced, and in other companies. [Results] We successfully reused some rules in both scenarios with minor adjustments, finding new defects to be fixed. The reuse of rules could discard from 58-90% of source code locations found by a naive search for the defects, reducing verification effort. However, the reused rules need adjustments to improve precision for defect localization, as precision ranged from 40-75%. Finally, we identified factors that have an impact on reusing custom rules. [Conclusions] We put forward that reusing customized static analysis rules can be beneficial, in particular when similarities in the architecture and programming style are observed. However, adjustment of the rules might be needed to enable effective reuse. We shared our insights and methodology on how to reuse custom static analysis rules properly.

@inproceedings{MendoncaSK20,
  author    = {Diogo S. Mendon\c{c}a and Arnd von Staa and Marcos Kalinowski},
  title     = {Pattern-Driven Maintenance: A Method to Prevent Unhandled Latent Exceptions in Web Applications},
  abstract  = {Background: Unhandled exceptions affect the reliability, usability, and security of web applications. Automatically detecting unhandled latent exceptions is difficult and application-specific. Hence, general approaches to deal with defects in web applications do not treat unhandled exceptions appropriately. Aims: To design and evaluate a method that can support finding, correcting, and preventing unhandled exceptions in web applications. Method: We designed a method called Pattern-Driven Maintenance (PDM), which relies on identifying defect patterns from failures and producing custom static analysis rules that can be used for prevention. We applied PDM to two industrial web applications measuring the reliability improvement, evaluated reuse of static analysis rules produced on within- and cross-company software, and studied the effectiveness, challenges faced, and acceptance of novice maintainers on applying PDM. Results: In both industry cases, we eliminated pattern-related failures improving the application reliability. Some of the static analysis rules produced by applying PDM were reused on within- and cross-company software. We identified knowledge and experiences that influence effectively applying the steps of the PDM method. Conclusions: PDM can help maintainers to improve the reliability of existing applications. We provide guidance on how to apply PDM, reuse the produced static analysis rules, and the knowledge and experiences needed to apply the PDM method effectively.},
  booktitle = {Proceedings of the XIX Brazilian Symposium on Software Quality, {SBQS'20 - CTDQS}, Brazil, December 1st - December 4th},
  pages     = {1-10},
  note      = {<font color="red">Summary Paper for the "Best Brazilian Software Quality PhD Thesis Award", received at SBQS 2020. Student: Diogo Mendon{\c{c}}a, Advisors: Arndt von Staa and Marcos Kalinowski</font>},
  year      = {2020},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/MendoncaSK20.pdf},
  doi       = {},
}

Background: Unhandled exceptions affect the reliability, usability, and security of web applications. Automatically detecting unhandled latent exceptions is difficult and application-specific. Hence, general approaches to deal with defects in web applications do not treat unhandled exceptions appropriately. Aims: To design and evaluate a method that can support finding, correcting, and preventing unhandled exceptions in web applications. Method: We designed a method called Pattern-Driven Maintenance (PDM), which relies on identifying defect patterns from failures and producing custom static analysis rules that can be used for prevention. We applied PDM to two industrial web applications measuring the reliability improvement, evaluated reuse of static analysis rules produced on within- and cross-company software, and studied the effectiveness, challenges faced, and acceptance of novice maintainers on applying PDM. Results: In both industry cases, we eliminated pattern-related failures improving the application reliability. Some of the static analysis rules produced by applying PDM were reused on within- and cross-company software. We identified knowledge and experiences that influence effectively applying the steps of the PDM method. Conclusions: PDM can help maintainers to improve the reliability of existing applications. We provide guidance on how to apply PDM, reuse the produced static analysis rules, and the knowledge and experiences needed to apply the PDM method effectively.

@inproceedings{KalinowskiLTCKIBPSW20,
  author    = {Marcos Kalinowski and Helio Lopes and Alex Furtado Teixeira and Gabriel da Silva Cardoso and Andr{\'e} Kuramoto and Bruno Itagyba and Solon Tarso Batista and Juliana Alves Pereira and Thuener Silva and Jorge Alam Warrak and Marcelo da Costa and Marinho Fischer and Cristiane Salgado and Bianca Teixeira and Jacques Chueke and Bruna Ferreira and Rodrigo Lima and Hugo Villamizar and Andr{\'e} Brand{\~a}o and Simone Barbosa and Marcus Poggi and Carlos Pelizaro and Deborah Lemes and Marcus Waltemberg and Odnei Lopes and Willer Goulart},
  title     = {Lean R&D: An Agile Research and Development Approach for Digital Transformation},
  abstract  = {Petrobras is a large publicly-held company that operates in the oil, gas and energy industry. Recently, they conducted internal dynamics to identify several Digital Transformation (DT) opportunities to leverage their operational excellence. Addressing such opportunities typically requires Research and Development (R&D) uncertainties that could lead traditional R&D cooperation terms to be negotiated in years. However, there are time-to-market constraints for fast-paced deliveries to experiment solution options. With this in mind, they partnered up with PUC-Rio to establish a new DT initiative. The goal of this paper is to present the Lean R&D approach, tailored within this new initiative, and results of two case studies regarding its application in practice. We designed Lean R&D integrating the following building blocks: (i) Lean Inceptions, to allow stakeholders to jointly outline a Minimal Viable Product (MVP); (ii) early parallel technical feasibility assessment and conception phases, allowing to ‘fail fast’; (iii) scrum-based development management; and (iv) strategically aligned continuous experimentation to test business hypotheses. In the two reported case studies, Lean R&D enabled addressing research-related uncertainties early and to efficiently deliver valuable MVPs within four months, showing itself suitable for supporting the DT initiative. Key success factors were the business strategy alignment, the defined roles and co-creation philosophy with strong integration between Petrobras and PUC-Rio’s teams, and continuous support of a highly qualified research team. Main opportunities for improvement, based on our lessons learned, rely on better adapting Lean Inceptions to the DT context and on scaling the approach to a project portfolio level of abstraction.},
  booktitle = {Product-Focused Software Process Improvement - 21st International Conference, {PROFES} 2020, Turin, Italy, November 25-27, 2020. Proceedings},
  pages     = {1-16},
  note      = {},          
  year      = {2020},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/KalinowskiLTCKIBPSW20.pdf},
  doi       = {10.1007/978-3-030-64148-1_7},
}

Petrobras is a large publicly-held company that operates in the oil, gas and energy industry. Recently, they conducted internal dynamics to identify several Digital Transformation (DT) opportunities to leverage their operational excellence. Addressing such opportunities typically requires Research and Development (R&D) uncertainties that could lead traditional R&D cooperation terms to be negotiated in years. However, there are time-to-market constraints for fast-paced deliveries to experiment solution options. With this in mind, they partnered up with PUC-Rio to establish a new DT initiative. The goal of this paper is to present the Lean R&D approach, tailored within this new initiative, and results of two case studies regarding its application in practice. We designed Lean R&D integrating the following building blocks: (i) Lean Inceptions, to allow stakeholders to jointly outline a Minimal Viable Product (MVP); (ii) early parallel technical feasibility assessment and conception phases, allowing to ‘fail fast’; (iii) scrum-based development management; and (iv) strategically aligned continuous experimentation to test business hypotheses. In the two reported case studies, Lean R&D enabled addressing research-related uncertainties early and to efficiently deliver valuable MVPs within four months, showing itself suitable for supporting the DT initiative. Key success factors were the business strategy alignment, the defined roles and co-creation philosophy with strong integration between Petrobras and PUC-Rio’s teams, and continuous support of a highly qualified research team. Main opportunities for improvement, based on our lessons learned, rely on better adapting Lean Inceptions to the DT context and on scaling the approach to a project portfolio level of abstraction.

@inproceedings{OliveiraLKVLS20,
  author    = {Rafael Pereira de Oliveira and Sergio Lifschitz and Marcos Kalinowski and Marx Viana and Carlos Lucena and Marcos Vaz Salles},
  title     = {Evaluating Database Self-Tuning Strategies in a Common Extensible Framework},
  abstract  = {Database automatic tuning tools are an essential class of database applications for database administrators (DBAs) and researchers. These self-management systems involve recurring and ubiquitous tasks, such as data extraction for workload acquisition and more specific features that depend on the tuning strategy, such as the specification of tuning action types and heuristics. Given the variety of approaches and implementations, it would be desirable to evaluate existing database self-tuning strategies, particularly recent and new heuristics, in a standard testbed. In this paper, we propose a reuse-oriented framework approach towards assessing and comparing automatic relational database tuning strategies. We employ our framework to instantiate three customized automated database tuning tools extended from our framework kernel, employing strategies using combinations of different tuning actions (indexes, partial indexes, and materialized views) for various RDBMS. Finally, we evaluate the effectiveness of these tools using a known database benchmark. Our results show that the framework enabled instantiating useful self-tuning tools with a low effort by just extending well-defined framework hot-spots. Hence, sharing it with the database research community may facilitate evolving state of the art on self-tunning strategies by enabling to evaluate different strategies on different RDBMS, serving as a common and extensible testbed.},
  booktitle = {Proceedings of the 34th Brazilian Symposium on Databases, {SBBD'20}, Brazil, September 28th - October 1st},
  pages     = {1-12},
  note      = {},          
  year      = {2020},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/OliveiraLKVLS20.pdf},
  doi       = {10.5753/sbbd.2020.13628},
}

Database automatic tuning tools are an essential class of database applications for database administrators (DBAs) and researchers. These self-management systems involve recurring and ubiquitous tasks, such as data extraction for workload acquisition and more specific features that depend on the tuning strategy, such as the specification of tuning action types and heuristics. Given the variety of approaches and implementations, it would be desirable to evaluate existing database self-tuning strategies, particularly recent and new heuristics, in a standard testbed. In this paper, we propose a reuse-oriented framework approach towards assessing and comparing automatic relational database tuning strategies. We employ our framework to instantiate three customized automated database tuning tools extended from our framework kernel, employing strategies using combinations of different tuning actions (indexes, partial indexes, and materialized views) for various RDBMS. Finally, we evaluate the effectiveness of these tools using a known database benchmark. Our results show that the framework enabled instantiating useful self-tuning tools with a low effort by just extending well-defined framework hot-spots. Hence, sharing it with the database research community may facilitate evolving state of the art on self-tunning strategies by enabling to evaluate different strategies on different RDBMS, serving as a common and extensible testbed.

@inproceedings{KalinowskiBLBPSVCTP20,
  author    = {Marcos Kalinowski and Solon Tarso Batista and Helio Lopes and Simone Barbosa and Marcus Poggi and Thuener Silva and Hugo Villamizar and Jacques Chueke and Bianca Teixeira and Juliana Alves Pereira and Bruna Ferreira and Rodrigo Lima and Gabriel da Silva Cardoso and Alex Furtado Teixeira and Jorge Alam Warrak and Marinho Fischer and Andr{\'e} Kuramoto and Bruno Itagyba and Cristiane Salgado and Carlos Pelizaro and Deborah Lemes and Marcelo Silva da Costa and Marcus Waltemberg and Odnei Lopes},
  title     = {Towards Lean R&D: An Agile Research and Development Approach for Digital Transformation},
  abstract  = {[Context] Petrobras is Brazil’s largest publicly-held company, operating in the oil, natural gas, and energy industry. Internal efforts enabled Petrobras to identify Digital Transformation (DT) opportunities to further promote their operational excellence. While addressing these opportunities typically requires Research and Development (R&D) uncertainties that could lead traditional R&D cooperation terms to be negotiated in years, there are time-to-market constraints for fast-paced deliveries to experiment solution options. Having this in mind, they partnered up with PUC-Rio to establish a new DT initiative. [Goal] The goal of this paper is to present the Lean R&D approach, tailored within the new initiative to meet the aforementioned DT needs. [Method] We designed Lean R&D integrating the following building blocks: (i) Lean Inceptions, to allow stakeholders to jointly outline a Minimal Viable Product (MVP); (ii) parallel technical feasibility assessment and conception phases, allowing to ‘fail fast’; (iii) scrum-based development management; and (iv) strategically aligned continuous experimentation to test business hypotheses. We report on first experiences of applying Lean R&D in practice. [Results] Lean R&D enabled addressing research-related uncertainties early and to efficiently deliver valuable MVPs within fast-paced four months cycles. [Conclusions] In our first experiences Lean R&D showed itself suitable for supporting DT initiatives. However, more formal case studies are needed. The business strategy alignment and the continuous support of a highly qualified research team were considered key success factors.},
  booktitle = {46th Euromicro Conference on Software Engineering and Advanced Applications, {SEAA} 2020, Portoroz, Slovenia, Aug 26-28},
  pages     = {132-136},
  note      = {},
  year      = {2020},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/KalinowskiBLBPSVCTP20.pdf},
  doi       = {10.1109/SEAA51224.2020.00030},
}

[Context] Petrobras is Brazil’s largest publicly-held company, operating in the oil, natural gas, and energy industry. Internal efforts enabled Petrobras to identify Digital Transformation (DT) opportunities to further promote their operational excellence. While addressing these opportunities typically requires Research and Development (R&D) uncertainties that could lead traditional R&D cooperation terms to be negotiated in years, there are time-to-market constraints for fast-paced deliveries to experiment solution options. Having this in mind, they partnered up with PUC-Rio to establish a new DT initiative. [Goal] The goal of this paper is to present the Lean R&D approach, tailored within the new initiative to meet the aforementioned DT needs. [Method] We designed Lean R&D integrating the following building blocks: (i) Lean Inceptions, to allow stakeholders to jointly outline a Minimal Viable Product (MVP); (ii) parallel technical feasibility assessment and conception phases, allowing to ‘fail fast’; (iii) scrum-based development management; and (iv) strategically aligned continuous experimentation to test business hypotheses. We report on first experiences of applying Lean R&D in practice. [Results] Lean R&D enabled addressing research-related uncertainties early and to efficiently deliver valuable MVPs within fast-paced four months cycles. [Conclusions] In our first experiences Lean R&D showed itself suitable for supporting DT initiatives. However, more formal case studies are needed. The business strategy alignment and the continuous support of a highly qualified research team were considered key success factors.

@inproceedings{LaignerKDBCLALZ20,
  author    = {Rodrigo Laigner and Marcos Kalinowski and Pedro Diniz and Leonardo Barros and Carlos Cassino and Melissa Lemos and Darlan Arruda and S{\'e}rgio Lifschitz and Yongluan Zhou},
  title     = {From a Monolithic Big Data System to a Microservices Event-Driven Architecture},
  abstract  = {[Context] Data-intensive systems, a.k.a. big data systems (BDS), are software systems that handle a large volume of data in the presence of performance quality attributes, such as scalability and availability. Before the advent of big data management systems (e.g. Cassandra) and frameworks (e.g. Spark), organizations had to cope with large data volumes with custom-tailored solutions. In particular, a decade ago, Tecgraf/PUC-Rio developed a system to monitor truck fleet in real-time and proactively detect events from the positioning data received. Over the years, the system evolved into a complex and large obsolescent code base involving a costly maintenance process. [Goal] We report our experience on replacing a legacy BDS with a microservice-based event-driven system. [Method] We applied action research, investigating the reasons that motivate the adoption of a microservice-based event-driven architecture, intervening to define the new architecture, and documenting the challenges and lessons learned. [Results] We perceived that the resulting architecture enabled easier maintenance and fault-isolation. However, the myriad of technologies and the complex data flow were perceived as drawbacks. Based on the challenges faced, we highlight opportunities to improve the design of big data reactive systems. [Conclusions] We believe that our experience provides helpful takeaways for practitioners modernizing systems with data-intensive requirements.},
  booktitle = {46th Euromicro Conference on Software Engineering and Advanced Applications, {SEAA} 2020, Portoroz, Slovenia, Aug 26-28},
  pages     = {213-220},
  note      = {},
  year      = {2020},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/LaignerKDBCLALZ20.pdf},
  doi       = {10.1109/SEAA51224.2020.00045},
}

[Context] Data-intensive systems, a.k.a. big data systems (BDS), are software systems that handle a large volume of data in the presence of performance quality attributes, such as scalability and availability. Before the advent of big data management systems (e.g. Cassandra) and frameworks (e.g. Spark), organizations had to cope with large data volumes with custom-tailored solutions. In particular, a decade ago, Tecgraf/PUC-Rio developed a system to monitor truck fleet in real-time and proactively detect events from the positioning data received. Over the years, the system evolved into a complex and large obsolescent code base involving a costly maintenance process. [Goal] We report our experience on replacing a legacy BDS with a microservice-based event-driven system. [Method] We applied action research, investigating the reasons that motivate the adoption of a microservice-based event-driven architecture, intervening to define the new architecture, and documenting the challenges and lessons learned. [Results] We perceived that the resulting architecture enabled easier maintenance and fault-isolation. However, the myriad of technologies and the complex data flow were perceived as drawbacks. Based on the challenges faced, we highlight opportunities to improve the design of big data reactive systems. [Conclusions] We believe that our experience provides helpful takeaways for practitioners modernizing systems with data-intensive requirements.

@inproceedings{MoyonARMK20,
  author    = {Fabiola Moyon and Pamela Almeida and Daniel Riofrio and Daniel Mendez and Marcos Kalinowski},
  title     = {Security Compliance in Agile Software Development: A Systematic Mapping Study},
  abstract  = {[Context] Systematic Literature Reviews (SLRs) have been adopted within the Software Engineering (SE) domain for more than a decade to provide meaningful summaries of evidence on several topics. Many of these SLRs are now outdated, and there are no standard proposals on how to update SLRs in SE. [Objective] The goal of this paper is to provide recommendations on how to best to search for  evidence when updating SLRs in SE. [Method] To achieve our goal, we compare and discuss outcomes from applying different search strategies to identifying primary studies in a previously published SLR update on effort estimation. [Results] The use of a single iteration forward snowballing with Google Scholar, and employing the original SLR and its primary studies as a seed set seems to be the most cost-effective way to search for new evidence when updating SLRs. [Conclusions] The recommendations can be used to support decisions on how to update SLRs in SE.},
  booktitle = {46th Euromicro Conference on Software Engineering and Advanced Applications, {SEAA} 2020, Portoroz, Slovenia, Aug 26-28},
  pages     = {413-420},
  note      = {},
  year      = {2020},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/MoyonARMK20.pdf},
  doi       = {10.1109/SEAA51224.2020.00073},
}

[Context] Systematic Literature Reviews (SLRs) have been adopted within the Software Engineering (SE) domain for more than a decade to provide meaningful summaries of evidence on several topics. Many of these SLRs are now outdated, and there are no standard proposals on how to update SLRs in SE. [Objective] The goal of this paper is to provide recommendations on how to best to search for evidence when updating SLRs in SE. [Method] To achieve our goal, we compare and discuss outcomes from applying different search strategies to identifying primary studies in a previously published SLR update on effort estimation. [Results] The use of a single iteration forward snowballing with Google Scholar, and employing the original SLR and its primary studies as a seed set seems to be the most cost-effective way to search for new evidence when updating SLRs. [Conclusions] The recommendations can be used to support decisions on how to update SLRs in SE.

@incollection{WagnerMFGK20,
  author = {Stefan Wagner and Daniel Mendez and Michael Felderer and Daniel Graziotin and Marcos Kalinowski},
  title = {Challenges in Survey Research},
  editor = {Guilherme Travassos and Michael Felderer},
  booktitle = {Contemporary Empirical Methods in Software Engineering},
  publisher = {Springer},
  year = {2020},
  pages = {93-125},
  note = {ISBN: 978-3-030-32488-9},
  chapter = {Challenges in Survey Research},
  urlAuthor_version = {http://www.inf.puc-rio.br/~kalinowski/publications/WagnerMFGK20.pdf},
  doi = {10.1007/978-3-030-32489-6_4}
}

@incollection{AlvesLNRLK20,
  author = {Paulo Henrique Alves and Rodrigo Laigner and Rafael Nasser and Gustavo Robichez and Helio Lopes and Marcos Kalinowski},
  title = {Desmistificando Blockchain: Conceitos e Aplica{\c{c}}ões},
  editor = {Cristiano Maciel and Jos{\'e} Viterbo},
  booktitle = {Computa{\c{c}}{\~a}o e Sociedade},
  publisher = {Sociedade Brasileira de Computa{\c{c}}{\~a}o},
  year = {2020},
  pages = {166-197},
  note = {ISBN: 978-6-555-88048-9},
  chapter = {22},
  urlAuthor_version = {http://www.inf.puc-rio.br/~kalinowski/publications/AlvesLNRLK20.pdf},
}

@article{WagnerEtAl19,
	author = {Stefan Wagner and Daniel M{\'{e}}ndez Fern{\'{a}}ndez and Michael Felderer and Antonio Vetro and Marcos Kalinowski and Roel Wieringa and Dietmar Pfahl and Tayana Conte and Marie-Therese Christiansson and Desmond Greer and Casper Lassenius and  Tomi M{\"{a}}nnist{\"{o}} and Maleknaz Nayebi and Markku Oivo and Birgit Penzenstadler and  Rafael Prikladnicki and Guenther Ruhe and Andr{\'{e}} Schekelmann and Sagar Sen and Rodrigo O. Sp{\'{\i}}nola and Ahmet Tuzcu and Jose Luis de la Vara and Dietmar Winkler},
	title = {Status Quo in Requirements Engineering: A Theory and a Global Family of Surveys},
	journal = {ACM Transactions on Software Engineering and Methdology},
	volume = {28},
	number = {2},
	note = {},      
	year = {2019},
	keywords = {},
	abstract = {Context: Requirements Engineering (RE) has established itself as a software engineering discipline over the past decades. While researchers have been investigating the RE discipline with a plethora of empirical studies, attempts to systematically derive an empirical theory in context of the RE discipline have just recently been started. However, such a theory is needed if we are to define and motivate guidance in performing high quality RE research and practice.
				Objective: We aim at providing an empirical and externally valid foundation for a theory of RE practice, which helps software engineers establish effective and efficient RE processes in a problem-driven manner.
				Method:We designed a survey instrument and an engineer-focused theory that was first piloted in Germany and, after making substantial modifications, has now been replicated in 10 countries world-wide. We have a theory in the form of a set of propositions inferred from our experiences and available studies, as well as the results from our pilot study in Germany. We evaluate the propositions with bootstrapped confidence intervals and derive potential explanations for the propositions.
				Results: In this article, we report on the design of the family of surveys, its underlying theory, and the full results obtained from the replication studies conducted in 10 countries with participants from 228 organisations. Our results represent a substantial step forward towards developing an empirical theory of RE practice. The results reveal, for example, that there are no strong differences between organisations in different countries and regions, that interviews, facilitated meetings and prototyping are the most used elicitation techniques, that requirements are often documented textually, that traces between requirements and code or design documents is common, that requirements specifications themselves are rarely changed and that requirements engineering (process) improvement endeavours are mostly internally driven.
				Conclusion: Our study establishes a theory that can be used as starting point for many further studies for more detailed investigations. Practitioners can use the results as theory-supported guidance on selecting suitable RE methods and techniques.},
	issn = {1049-331X},	
	pages = {9:1-9:48},	
	doi = {10.1145/3306607},
	urlAuthor_version = {http://www.inf.puc-rio.br/~kalinowski/publications/WagnerEtAl19.pdf},
}

Context: Requirements Engineering (RE) has established itself as a software engineering discipline over the past decades. While researchers have been investigating the RE discipline with a plethora of empirical studies, attempts to systematically derive an empirical theory in context of the RE discipline have just recently been started. However, such a theory is needed if we are to define and motivate guidance in performing high quality RE research and practice. Objective: We aim at providing an empirical and externally valid foundation for a theory of RE practice, which helps software engineers establish effective and efficient RE processes in a problem-driven manner. Method:We designed a survey instrument and an engineer-focused theory that was first piloted in Germany and, after making substantial modifications, has now been replicated in 10 countries world-wide. We have a theory in the form of a set of propositions inferred from our experiences and available studies, as well as the results from our pilot study in Germany. We evaluate the propositions with bootstrapped confidence intervals and derive potential explanations for the propositions. Results: In this article, we report on the design of the family of surveys, its underlying theory, and the full results obtained from the replication studies conducted in 10 countries with participants from 228 organisations. Our results represent a substantial step forward towards developing an empirical theory of RE practice. The results reveal, for example, that there are no strong differences between organisations in different countries and regions, that interviews, facilitated meetings and prototyping are the most used elicitation techniques, that requirements are often documented textually, that traces between requirements and code or design documents is common, that requirements specifications themselves are rarely changed and that requirements engineering (process) improvement endeavours are mostly internally driven. Conclusion: Our study establishes a theory that can be used as starting point for many further studies for more detailed investigations. Practitioners can use the results as theory-supported guidance on selecting suitable RE methods and techniques.

@inproceedings{LaignerLKPS19,
 author = {Rodrigo Laigner and Sergio Lifschitz and Marcos Kalinowski and Marcus Poggi and Marcos Vaz Salles},
 title = {Towards a Technique for Extracting Relational Actors from Monolithic Applications},
 abstract = {Relational actors, or reactors for short, integrate the actor model with the relational data model, providing an abstraction for enabling actor-relational database systems. However, as a novel model of computation for databases, there is no extensive work on reasoning about reactor modeling. To close this gap, this paper aims to propose as well as evaluate a technique to extract reactors from a monolithic system. For evaluation, we selected a REST-based open-source OLTP system in which a decomposition to microservices was conducted and applied our technique on its predecessor monolithic version. Our technique led to the same set of decisions, regarding table and behavior selection, taken by experts when decomposing the same system into microservices. The proposed technique can be seen as a first step towards supporting practitioners in decomposing OLTP systems into reactors.},
 booktitle = {Proceedings of the 34th Brazilian Symposium on Databases, {SBBD'19}, Fortaleza, Brazil, October 7-10},
 year = {2019},
 pages = {133-144},
 urlAuthor_version = {http://www.inf.puc-rio.br/~kalinowski/publications/LaignerLKPS19.pdf},
 doi = {10.5753/sbbd.2019.8814},
} 

Relational actors, or reactors for short, integrate the actor model with the relational data model, providing an abstraction for enabling actor-relational database systems. However, as a novel model of computation for databases, there is no extensive work on reasoning about reactor modeling. To close this gap, this paper aims to propose as well as evaluate a technique to extract reactors from a monolithic system. For evaluation, we selected a REST-based open-source OLTP system in which a decomposition to microservices was conducted and applied our technique on its predecessor monolithic version. Our technique led to the same set of decisions, regarding table and behavior selection, taken by experts when decomposing the same system into microservices. The proposed technique can be seen as a first step towards supporting practitioners in decomposing OLTP systems into reactors.

@inproceedings{VillamizarAKGM19,
 author = {Hugo Villamizar and Amadeu Anderlin-Neto and Marcos Kalinowski and Alessandro Garcia and Daniel M{\'{e}}ndez Fern{\'{a}}ndez},
 title = {An Approach for Reviewing Security-Related Aspects in Agile Requirements Specifications of Web Applications},
 abstract = {Defects in requirements specifications can have severe consequences during the software development lifecycle. Some of them result in overall project failure due to incorrect or missing quality characteristics. Security is one of those quality characteristics that need to be considered in early phases. There are several concerns that make security difficult to deal with; for instance, (1) when stakeholders discuss general requirements in (review) meetings, they are often not aware that they should also discuss security-related topics, and in the rather rare cases they are aware (2), they typically do not have sufficient security expertise. This picture is even more challenging in agile development contexts, where lightweight documentation are typically involved. To address these issues, we designed an approach that considers user stories and security specifications as input and relates those user stories to security properties via Natural Language Processing (NLP) techniques. Based on the related security properties, our approach then identifies high-level security requirements from the Open Web Application Security Project (OWASP) to be verified afterwards. In a last step, the verification of the generated security requirements is then conducted via a focused reading technique. We finally validate our approach via a controlled experiment comparing the effectiveness and efficiency of novice inspectors (we used two different groups of students) verifying security aspects in agile requirements using our generated reading techniques against using the complete list of OWASP high level security requirements and a the same list of defect types embedded in our technique. The (statistically significant) results indicate that using the reading technique has a positive impact (with large effect size) on the performance of inspectors in terms of effectiveness and efficiency.},
 booktitle = {Proceedings of the 27th {IEEE} International Requirements Engineering Conference, {RE'19}, Jeju Island, South Korea, September 23-27},
 year = {2019},
 note = {<font color="red">Nominated for Distinguished Paper Award at RE'19! Paper used as basis for receiving the "Second Best Brazilian Software Quality M.Sc. Dissertation Award" at SBQS 2020 - Student: Hugo Villamizar, Advisor: Marcos Kalinowski</font>}, 
 pages = {86-97},
 urlAuthor_version = {http://www.inf.puc-rio.br/~kalinowski/publications/VillamizarAKGM19.pdf},
 doi = {10.1109/RE.2019.00020},
} 

Defects in requirements specifications can have severe consequences during the software development lifecycle. Some of them result in overall project failure due to incorrect or missing quality characteristics. Security is one of those quality characteristics that need to be considered in early phases. There are several concerns that make security difficult to deal with; for instance, (1) when stakeholders discuss general requirements in (review) meetings, they are often not aware that they should also discuss security-related topics, and in the rather rare cases they are aware (2), they typically do not have sufficient security expertise. This picture is even more challenging in agile development contexts, where lightweight documentation are typically involved. To address these issues, we designed an approach that considers user stories and security specifications as input and relates those user stories to security properties via Natural Language Processing (NLP) techniques. Based on the related security properties, our approach then identifies high-level security requirements from the Open Web Application Security Project (OWASP) to be verified afterwards. In a last step, the verification of the generated security requirements is then conducted via a focused reading technique. We finally validate our approach via a controlled experiment comparing the effectiveness and efficiency of novice inspectors (we used two different groups of students) verifying security aspects in agile requirements using our generated reading techniques against using the complete list of OWASP high level security requirements and a the same list of defect types embedded in our technique. The (statistically significant) results indicate that using the reading technique has a positive impact (with large effect size) on the performance of inspectors in terms of effectiveness and efficiency.

@inproceedings{LaignerKCMG19,
 author = {Rodrigo Laigner and Marcos Kalinowski and Luiz Carvalho and Diogo Mendonca and Alessandro Garcia},
 title = {Towards a Catalog of Java Dependency Injection Anti-Patterns},
 abstract = {[Context] Dependency Injection (DI) is a commonly applied mechanism to decouple classes from their dependencies in order to provide better modularization of software. In the context of Java, the availability of a DI specification and popular frameworks, such as Spring, facilitate DI usage in software projects. However, bad DI implementation practices can have negative consequences. Even though the literature suggests the existence of DI anti-patterns, there is no detailed catalog of such bad practices. Moreover, there is no evidence on their occurrence and perceived usefulness from the developer’s point of view. [Goal] Our goal is to review the reported DI anti-patterns in order to analyze their completeness and to propose and evaluate a novel catalog of DI anti-patterns in the context of Java. [Method] We propose an initial catalog containing twelve Java DI anti-patterns. We selected four open source software projects that adopt a DI framework and developed a tool to statically analyze the occurrence of the DI anti-patterns within their source code. Also, we conducted a survey through face to face interviews with three experienced developers that regularly apply DI. [Results] At least nine different DI anti-patterns appeared in each analyzed project. In addition, the feedback received from the developers confirmed their relevance and the importance of investing further effort towards a catalog. [Conclusion] The results indicate that the initial catalog contains Java DI anti-patterns that occur in practice and are useful. Sharing it with practitioners may help them to avoid such anti-patterns. Sharing it with the research community will enable further improving the catalog.},
 booktitle = {Proceedings of the 33rd Brazilian Symposium on Software Engineering, {SBES'19}, Salvador, Brazil, September 23-27},
 year = {2019},
 note = {<font color="red">SBES 2019 Third Best Paper Award!</font>}, 
 pages = {104-113},
 urlAuthor_version = {http://www.inf.puc-rio.br/~kalinowski/publications/LaignerKCMG19.pdf},
 doi = {10.1145/3350768.3350771},
} 

[Context] Dependency Injection (DI) is a commonly applied mechanism to decouple classes from their dependencies in order to provide better modularization of software. In the context of Java, the availability of a DI specification and popular frameworks, such as Spring, facilitate DI usage in software projects. However, bad DI implementation practices can have negative consequences. Even though the literature suggests the existence of DI anti-patterns, there is no detailed catalog of such bad practices. Moreover, there is no evidence on their occurrence and perceived usefulness from the developer’s point of view. [Goal] Our goal is to review the reported DI anti-patterns in order to analyze their completeness and to propose and evaluate a novel catalog of DI anti-patterns in the context of Java. [Method] We propose an initial catalog containing twelve Java DI anti-patterns. We selected four open source software projects that adopt a DI framework and developed a tool to statically analyze the occurrence of the DI anti-patterns within their source code. Also, we conducted a survey through face to face interviews with three experienced developers that regularly apply DI. [Results] At least nine different DI anti-patterns appeared in each analyzed project. In addition, the feedback received from the developers confirmed their relevance and the importance of investing further effort towards a catalog. [Conclusion] The results indicate that the initial catalog contains Java DI anti-patterns that occur in practice and are useful. Sharing it with practitioners may help them to avoid such anti-patterns. Sharing it with the research community will enable further improving the catalog.

@inproceedings{BibianoFOGKFOOC19,
 author = {Ana Carla Bibiano and Eduardo Fernandes and Daniel Oliveira and Alessandro Garcia and Marcos Kalinowski and Baldoino Fonseca and Roberto Oliveira and Anderson Oliveira and Diego Cedrim},
 title = {A Quantitative Study on Characteristics and Effect of Batch Refactoring on Code Smells},
 abstract = {Background: Code refactoring aims to improve code structures via code transformations. A single transformation rarely suffices to fully remove code smells that reveal poor code structures. Most transformations are applied in batches, i.e. sets of interrelated transformations, rather than in isolation. Nevertheless, empirical knowledge on batch application, or batch refactoring, is scarce. Such scarceness helps little to improve current refactoring practices. Aims: We analyzed 57 open and closed software projects. We aimed to understand batch application from two perspectives: characteristics that typically constitute a batch (e.g., the variety of transformation types employed), and the batch effect on smells. Method: We analyzed 19 smell types and 13 transformation types. We identified 4,607 batches, each applied by the same developer on the same code element (method or class); we expected to have batches whose transformations are closely interrelated. We computed (1) the frequency in which five batch characteristic manifest, (2) the probability of each batch characteristics to remove smells, and (3) the frequency in which batches introduce and remove smells. Results: Most batches are quite simple: although most batches are applied on more than one method (90%), they are usually composed of the same transformation type (72%) and only two transformations (57%). Batches applied on a single method are 2.6 times more prone to fully remove smells than batches affecting more than one method. Surprisingly, batches mostly ended up introducing (51%) or not fully removing (38%) smells. Conclusions: The batch simplicity suggests that developers have sub-explored the combinations of transformations within a batch. We summarized some batches that may fully remove smells, so that developers can incorporate them into current refactoring practices.},
 booktitle = {Proceedings of the 12th {ACM/IEEE} International Symposium on Empirical Software Engineering and Measurement, {ESEM} 2019, Porto de Galinhas, Brazil, September 19-20},
 year = {2019},
 pages = {1-11},
 urlAuthor_version = {http://www.inf.puc-rio.br/~kalinowski/publications/BibianoFOGKFOOC19.pdf},
 doi = {10.1109/ESEM.2019.8870183},
} 

Background: Code refactoring aims to improve code structures via code transformations. A single transformation rarely suffices to fully remove code smells that reveal poor code structures. Most transformations are applied in batches, i.e. sets of interrelated transformations, rather than in isolation. Nevertheless, empirical knowledge on batch application, or batch refactoring, is scarce. Such scarceness helps little to improve current refactoring practices. Aims: We analyzed 57 open and closed software projects. We aimed to understand batch application from two perspectives: characteristics that typically constitute a batch (e.g., the variety of transformation types employed), and the batch effect on smells. Method: We analyzed 19 smell types and 13 transformation types. We identified 4,607 batches, each applied by the same developer on the same code element (method or class); we expected to have batches whose transformations are closely interrelated. We computed (1) the frequency in which five batch characteristic manifest, (2) the probability of each batch characteristics to remove smells, and (3) the frequency in which batches introduce and remove smells. Results: Most batches are quite simple: although most batches are applied on more than one method (90%), they are usually composed of the same transformation type (72%) and only two transformations (57%). Batches applied on a single method are 2.6 times more prone to fully remove smells than batches affecting more than one method. Surprisingly, batches mostly ended up introducing (51%) or not fully removing (38%) smells. Conclusions: The batch simplicity suggests that developers have sub-explored the combinations of transformations within a batch. We summarized some batches that may fully remove smells, so that developers can incorporate them into current refactoring practices.

@inproceedings{LageKTS19,
 author = {Luiz Carlos da Fonseca Lage and Marcos Kalinowski and Daniela Trevisan and Rodrigo Spinola},
 title = {Usability Technical Debt in Software Projects: A Multi-Case Study},
 abstract = {Background: Over the years, several studies were conducted aiming at understanding the Technical Debt (TD) phenomenon and its implications on software development. Most of these studies focus on source code related TD types. The absence of empirical studies on usability debt motivated our research. Aims: The goal of this paper is to provide an initial usability debt characterization in software projects regarding its occurrence, type, and resolution effort. Method: We conducted a multi-case study, analyzing TD items of five software projects from four different companies. Results: We identified and classified 145 TD items in the projects. The analysis of these items allowed us to observe that the frequency of usability TD items ranged from 10.4% to 20.8% in the projects. The usability debt items cover a range of usability issues, violating eight out of the ten Nielsen usability heuristics. Regarding effort for paying the TD, usability debt items require a relatively low effort, ranging from 5.1% to 6.7% of the total TD resolution effort in the analyzed projects. Conclusions: Usability TD items are frequent, concern relevant usability issues and typically require low effort for their payment. Hence, paying this type of TD should receive high priority in TD management strategies.},
 booktitle = {Proceedings of the 13th {ACM/IEEE} International Symposium on Empirical Software Engineering and Measurement, {ESEM} 2019, Porto de Galinhas, Brazil, September 19-20},
 year = {2019},
 pages = {1-6},
 urlAuthor_version = {http://www.inf.puc-rio.br/~kalinowski/publications/LageKTS19.pdf},
 doi = {10.1109/ESEM.2019.8870180},
} 

Background: Over the years, several studies were conducted aiming at understanding the Technical Debt (TD) phenomenon and its implications on software development. Most of these studies focus on source code related TD types. The absence of empirical studies on usability debt motivated our research. Aims: The goal of this paper is to provide an initial usability debt characterization in software projects regarding its occurrence, type, and resolution effort. Method: We conducted a multi-case study, analyzing TD items of five software projects from four different companies. Results: We identified and classified 145 TD items in the projects. The analysis of these items allowed us to observe that the frequency of usability TD items ranged from 10.4% to 20.8% in the projects. The usability debt items cover a range of usability issues, violating eight out of the ten Nielsen usability heuristics. Regarding effort for paying the TD, usability debt items require a relatively low effort, ranging from 5.1% to 6.7% of the total TD resolution effort in the analyzed projects. Conclusions: Usability TD items are frequent, concern relevant usability issues and typically require low effort for their payment. Hence, paying this type of TD should receive high priority in TD management strategies.

@inproceedings{MendesFWK19,
  author    = {Emilia Mendes and Katia Felizardo and Claes Wohlin and Marcos Kalinowski},
  title     = {Search Strategy to Update Systematic Literature Reviews in Software Engineering},
  abstract  = {[Context] Systematic Literature Reviews (SLRs) have been adopted within the Software Engineering (SE) domain for more than a decade to provide meaningful summaries of evidence on several topics. Many of these SLRs are now outdated, and there are no standard proposals on how to update SLRs in SE. [Objective] The goal of this paper is to provide recommendations on how to best to search for  evidence when updating SLRs in SE. [Method] To achieve our goal, we compare and discuss outcomes from applying different search strategies to identifying primary studies in a previously published SLR update on effort estimation. [Results] The use of a single iteration forward snowballing with Google Scholar, and employing the original SLR and its primary studies as a seed set seems to be the most cost-effective way to search for new evidence when updating SLRs. [Conclusions] The recommendations can be used to support decisions on how to update SLRs in SE.},
  booktitle = {45th Euromicro Conference on Software Engineering and Advanced Applications, {SEAA} 2019, Kallithea, Greece, Aug 28-30},
  pages     = {355-362},
  note      = {<font color="red">SEAA 2019 Best Paper Award!</font>},          
  year      = {2019},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/MendesFWK19.pdf},
  doi       = {10.1109/SEAA.2019.00061},
}

[Context] Systematic Literature Reviews (SLRs) have been adopted within the Software Engineering (SE) domain for more than a decade to provide meaningful summaries of evidence on several topics. Many of these SLRs are now outdated, and there are no standard proposals on how to update SLRs in SE. [Objective] The goal of this paper is to provide recommendations on how to best to search for evidence when updating SLRs in SE. [Method] To achieve our goal, we compare and discuss outcomes from applying different search strategies to identifying primary studies in a previously published SLR update on effort estimation. [Results] The use of a single iteration forward snowballing with Google Scholar, and employing the original SLR and its primary studies as a seed set seems to be the most cost-effective way to search for new evidence when updating SLRs. [Conclusions] The recommendations can be used to support decisions on how to update SLRs in SE.

@inproceedings{AnderlinNetoKGWB19,
  author    = {Amadeu Anderlin-Neto and
               Marcos Kalinowski and
               Alessandro Garcia and
               Dietmar Winkler and
               Stefan Biffl},
  title     = {A Preliminary Comparison of Using Variability Modeling Approaches to Represent Experiment Families},
  abstract  = {Background: Replication is essential to build knowledge in empirical science. Experiment replications reported in the software engineering context present variabilities on their design elements, e.g., variables, materials. The understanding of these variabilities is required to plan experimental replications within a research program. However, the lack of an explicit representation of experiments’ variabilities and commonalities is likely to hamper their understanding and replication planning. Aims: The goal of this paper is to explore the use of Variability Modeling Approaches (VMAs) to represent experiment families (i.e., an original study and its replications) and to investigate the feasibility of using VMAs to support experiment replication planning. Method: We selected two experiment families, analyzed their commonalities and variabilities, and represented them using a set of well-known VMAs: Feature Model, Decision Model, and Orthogonal Variability Model. Based on the resulting models, we conducted a preliminary comparison of using such alternative VMAs to support replication planning. Results: Subjects were able to plan consistent experiment replications with the VMAs as support. Additionally, through a qualitative analysis, we identified and discuss advantages and limitations of using the VMAs. Conclusions: It is feasible to represent experiment families and to plan replications using VMAs. Based on our emerging results, we conclude that the Feature Model VMA provides the most suitable representation. Furthermore, we identified benefits in a potential merge between the Feature Model and Decision Model VMAs to provide more details to support replication planning.},
  booktitle = {23rd International Conference on Evaluation and Assessment in Software
               Engineering, {EASE} '19, Copenhagen, Denmark, April 14-17,
               2019},
  pages     = {333-338},
  year      = {2019},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/AnderlinNetoKGWB19.pdf},
  doi       = {10.1145/3319008.3319356},
}

Background: Replication is essential to build knowledge in empirical science. Experiment replications reported in the software engineering context present variabilities on their design elements, e.g., variables, materials. The understanding of these variabilities is required to plan experimental replications within a research program. However, the lack of an explicit representation of experiments’ variabilities and commonalities is likely to hamper their understanding and replication planning. Aims: The goal of this paper is to explore the use of Variability Modeling Approaches (VMAs) to represent experiment families (i.e., an original study and its replications) and to investigate the feasibility of using VMAs to support experiment replication planning. Method: We selected two experiment families, analyzed their commonalities and variabilities, and represented them using a set of well-known VMAs: Feature Model, Decision Model, and Orthogonal Variability Model. Based on the resulting models, we conducted a preliminary comparison of using such alternative VMAs to support replication planning. Results: Subjects were able to plan consistent experiment replications with the VMAs as support. Additionally, through a qualitative analysis, we identified and discuss advantages and limitations of using the VMAs. Conclusions: It is feasible to represent experiment families and to plan replications using VMAs. Based on our emerging results, we conclude that the Feature Model VMA provides the most suitable representation. Furthermore, we identified benefits in a potential merge between the Feature Model and Decision Model VMAs to provide more details to support replication planning.

@inproceedings{GracioliAKOSWB19,
  author    = {Carlos Gracioli-Neto and
               Amadeu Anderlin-Neto and
               Marcos Kalinowski and
               Daniel Oliveira and
               Marta Sabou and
               Dietmar Winkler and			   
               Stefan Biffl},
  title     = {Using Model Scoping with Expected Model Elements to Support Software Model Inspections: Results of a Controlled Experiment},
  abstract  = {Context: Software inspection represents an effective way to identify defects in early phase software artifacts, such as models. Unfortunately, large models and associated reference documents cannot be thoroughly inspected in one inspection session of typically up to two hours. Considerably longer sessions have shown a much lower defect detection efficiency due to cognitive fatigue. Goal: The goal of this paper is to propose and evaluate a Model Scoping approach to allow inspecting specific parts of interest in large models. Method: First, we designed the approach, which involves identifying Expected Model Elements (EMEs) in selected parts of the reference document and then using these EMEs to scope the model (i.e., remove unrelated parts). These EMEs can also be used to support inspectors during detect defection. We conducted a controlled experiment using industrial artifacts. Subjects were asked to conduct UML class diagram inspections based on selected parts of functional specifications. In the experimental treatment, Model Scoping was applied and inspectors were provided with the scoped model and the EMEs. The control group used the original model directly, without EMEs. We measured the inspectors’ defect detection effectiveness and efficiency and collected qualitative data on the perceived complexity. Results: Applying Model Scoping prior to the inspection significantly increased the inspector defect detection effectiveness and efficiency, with large effect sizes. Qualitative data allowed observing a perception of reduced complexity during the inspection. Conclusion: Being able to effectively and efficiently inspect large models against selected parts of reference documents is a practical need, in particular in the context of incremental and agile process models. The experiment showed promising results for supporting such inspections using the proposed Model Scoping approach.},
  booktitle = {{ICEIS} 2019 - Proceedings of the 21st International Conference on Enterprise Information Systems, Crete, Greece, May 3-5,
               2019},
  pages     = {107-118},
  note      = {},        
  year      = {2019},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/GracioliAKOSWB19.pdf},
  doi       = {10.5220/0007691001070118},
}

Context: Software inspection represents an effective way to identify defects in early phase software artifacts, such as models. Unfortunately, large models and associated reference documents cannot be thoroughly inspected in one inspection session of typically up to two hours. Considerably longer sessions have shown a much lower defect detection efficiency due to cognitive fatigue. Goal: The goal of this paper is to propose and evaluate a Model Scoping approach to allow inspecting specific parts of interest in large models. Method: First, we designed the approach, which involves identifying Expected Model Elements (EMEs) in selected parts of the reference document and then using these EMEs to scope the model (i.e., remove unrelated parts). These EMEs can also be used to support inspectors during detect defection. We conducted a controlled experiment using industrial artifacts. Subjects were asked to conduct UML class diagram inspections based on selected parts of functional specifications. In the experimental treatment, Model Scoping was applied and inspectors were provided with the scoped model and the EMEs. The control group used the original model directly, without EMEs. We measured the inspectors’ defect detection effectiveness and efficiency and collected qualitative data on the perceived complexity. Results: Applying Model Scoping prior to the inspection significantly increased the inspector defect detection effectiveness and efficiency, with large effect sizes. Qualitative data allowed observing a perception of reduced complexity during the inspection. Conclusion: Being able to effectively and efficiently inspect large models against selected parts of reference documents is a practical need, in particular in the context of incremental and agile process models. The experiment showed promising results for supporting such inspections using the proposed Model Scoping approach.

@inproceedings{MendezFSFGKVWBL19,
  author    = {Daniel M{\'{e}}ndez Fern{\'{a}}ndez and Xavier Franch and Norbert Seyff and Michael Felderer and Martin Glinz and Marcos Kalinowski and Andreas Vogelsang and Stefan Wagner and Stan Bühne and Kim Lauenroth},
  title     = {Do We Preach What We Practice? Investigating the Practical Relevance of Requirements Engineering Syllabi – The IREB Case},
  abstract  = {Nowadays, there exist a plethora of different educational syllabi for Requirements Engineering (RE), all aiming at incorporating practically relevant educational units (EUs). Many of these syllabi are based, in one way or the other, on the syllabi provided by the International Requirements Engineering Board (IREB), a non-profit organisation devoted to standardised certification programs for RE. IREB syllabi are developed by RE experts and are, thus, based on the assumption that they address topics of practical relevance. However, little is known about to what extent practitioners actually perceive those contents as useful. We have started a study to investigate the relevance of the EUs included in the IREB Foundation Level certification programme. In the first phase reported in this paper, we have surveyed practitioners mainly from DACH countries (Germany, Austria and Switzerland) who participated in the IREB certification. Later phases will widen the scope both by including other countries and by not requiring IREB-certified participants. The results shall foster a critical reflection on the practical relevance of EUs built upon the de-facto standard syllabus of IREB.},
  booktitle = {CIbSE 2019 - XXII Ibero-American Conference on Software Engineering, Havana, Cuba, April 22-26},
  pages={476-487},  
  year={2019},
  note = {<font color="red">CIBSE RET 2019 Second Best Paper Award!</font>},
  urlAuthor_version={http://www.inf.puc-rio.br/~kalinowski/publications/MendezFSFGKVWBL19.pdf},
}

Nowadays, there exist a plethora of different educational syllabi for Requirements Engineering (RE), all aiming at incorporating practically relevant educational units (EUs). Many of these syllabi are based, in one way or the other, on the syllabi provided by the International Requirements Engineering Board (IREB), a non-profit organisation devoted to standardised certification programs for RE. IREB syllabi are developed by RE experts and are, thus, based on the assumption that they address topics of practical relevance. However, little is known about to what extent practitioners actually perceive those contents as useful. We have started a study to investigate the relevance of the EUs included in the IREB Foundation Level certification programme. In the first phase reported in this paper, we have surveyed practitioners mainly from DACH countries (Germany, Austria and Switzerland) who participated in the IREB certification. Later phases will widen the scope both by including other countries and by not requiring IREB-certified participants. The results shall foster a critical reflection on the practical relevance of EUs built upon the de-facto standard syllabus of IREB.

@inproceedings{MendozaSKIM19,
  author    = {Isela Mendoza and               
               U{\'e}verton Souza and
			   Marcos Kalinowski and
			   Ruben Interian and
               Leonardo Murta},   
  title     = {An Efficient Algorithm for Combining Verification and Validation Methods},
  abstract  = {An adequate combination of verification and validation (V&V) methods is important to improve software quality control throughout the development process and to reduce costs. However, to find an appropriate set of V&V methods that properly addresses the desired quality characteristics of a given project is a NP-hard problem. In this paper, we present a novel approach that combines V&V methods efficiently in order to properly cover a set of quality characteristics. We modelled the problem using a bipartite graph to represent the relationships between V&V methods and quality characteristics. Then we interpreted our problem as the Set Cover problem. Although Set Cover is considered hard to be solved, through the theoretical framework of Parameterized Complexity we propose an FPT-Algorithm (fixed-parameter tractable algorithm) that effectively solves the problem, considering the number of quality characteristics to be covered as a fixed parameter. We conclude that the proposed algorithm enables combining V&V methods in a scalable and efficient way, representing a valuable contribution to the community.},
  booktitle = {45th International Conference on Current Trends in Theory and Practice of Computer Science, {SOFSEM} 2019, Novy Smokovec, Slovakia, January 27-30},
  pages     = {324-340},
  year      = {2019},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/MendozaSKIM19.pdf},
  doi       = {10.1007/978-3-030-10801-4_26},
}

An adequate combination of verification and validation (V&V) methods is important to improve software quality control throughout the development process and to reduce costs. However, to find an appropriate set of V&V methods that properly addresses the desired quality characteristics of a given project is a NP-hard problem. In this paper, we present a novel approach that combines V&V methods efficiently in order to properly cover a set of quality characteristics. We modelled the problem using a bipartite graph to represent the relationships between V&V methods and quality characteristics. Then we interpreted our problem as the Set Cover problem. Although Set Cover is considered hard to be solved, through the theoretical framework of Parameterized Complexity we propose an FPT-Algorithm (fixed-parameter tractable algorithm) that effectively solves the problem, considering the number of quality characteristics to be covered as a fixed parameter. We conclude that the proposed algorithm enables combining V&V methods in a scalable and efficient way, representing a valuable contribution to the community.

@inproceedings{MendozaKSF19,
  author    = {Isela Mendoza and
               Marcos Kalinowski and
               U{\'e}verton Souza and
               Michael Felderer},   
  title     = {Relating Verification and Validation Methods to Software Product Quality Characteristics: Results of an Expert Survey},
  abstract  = {[Context] Employing appropriate verification and validation (V&V) methods is essential to improve software product quality. However, while several V&V methods have been documented, little is known about how these methods relate to specific product quality characteristics. [Goal] The goal of this paper is to provide an initial understanding on the suitability of selected V&V methods to address ISO 25010 software product quality characteristics. [Method] Therefore, we compiled a list of V&V methods and conducted a survey with V&V experts, asking them to evaluate how well each V&V method allows addressing the ISO 25010 characteristics. [Results] We received 19 answers from experts of 7 different countries. Our results express the aggregated expert opinion. It is noteworthy that the experts mostly agreed in their opinions, indicating consistency in the results. [Conclusions] To the best of our knowledge this is the first result on the relationship between V&V methods and quality characteristics. We believe that the aggregated opinion of 19 experts can serve as a starting point for further investigations by other researchers and to provide an initial understanding to practitioners.},
  booktitle = {Software Quality Days,
               11th International Conference, {SWQD} 2019, Vienna, Austria, January 15-18},
  pages     = {33-44},
  year      = {2019},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/MendozaKSF19.pdf},
  doi       = {10.1007/978-3-030-05767-1_3},
}

[Context] Employing appropriate verification and validation (V&V) methods is essential to improve software product quality. However, while several V&V methods have been documented, little is known about how these methods relate to specific product quality characteristics. [Goal] The goal of this paper is to provide an initial understanding on the suitability of selected V&V methods to address ISO 25010 software product quality characteristics. [Method] Therefore, we compiled a list of V&V methods and conducted a survey with V&V experts, asking them to evaluate how well each V&V method allows addressing the ISO 25010 characteristics. [Results] We received 19 answers from experts of 7 different countries. Our results express the aggregated expert opinion. It is noteworthy that the experts mostly agreed in their opinions, indicating consistency in the results. [Conclusions] To the best of our knowledge this is the first result on the relationship between V&V methods and quality characteristics. We believe that the aggregated opinion of 19 experts can serve as a starting point for further investigations by other researchers and to provide an initial understanding to practitioners.

@article{OliveiraTGPCMORKSMB18,
	author = {Alessandreia Oliveira and Gabriel Tessarolli and Gleiph Ghiotto and Bruno Pinto and Fernando Campello and Matheus Marques and Carlos Oliveira and Igor Rodrigues and Marcos Kalinowski and U{\'e}verton Souza and Leonardo Murta and Vanessa Braganholo},
	title = {An Efficient Similarity-based Approach for Comparing XML Documents},
	journal = {Information Systems},
	volume = {78},
	number = {},
	note = {},      
	year = {2018},
	keywords = {},
	abstract = {XML documents are widely used to interchange information among heterogeneous systems, ranging from office applications to scientific experiments. Independently of the domain, XML documents may evolve over time, so identifying and understanding the changes they undergo becomes crucial. Some syntactic-based diff approaches have been proposed to address this problem. They are mainly designed to compare revisions of XML documents using explicit IDs to match elements. However, elements in different revisions may not share IDs due to tool incompatibility or even divergent or missing schemas. In this paper, we present Phoenix, a similarity-based approach for comparing revisions of XML documents that does not rely on explicit IDs. Phoenix uses dynamic programming and optimization algorithms to compare different features (e.g., element name, attributes, content, and sub-elements) of XML documents and calculate the similarity degree between them. We compared Phoenix with X-Diff and XyDiff, two state-of-the-art XML diff algorithms. XyDiff was the fastest approach, but failed in providing precise matching results. X-Diff presented higher efficacy in 30 of the 56 scenarios, but was slow. Phoenix performed in a fraction of the running time required by X-Diff, and achieved the best results in terms of efficacy in 26 of 56 tested scenarios. In our evaluations Phoenix was by far the most efficient approach to match elements across revisions of the same XML document.},
	issn = {0306-4379},	
	pages = {40-57},	
	doi = {10.1016/j.is.2018.07.001},
	urlAuthor_version = {http://www.inf.puc-rio.br/~kalinowski/publications/OliveiraTGPCMORKSMB18.pdf},
}

XML documents are widely used to interchange information among heterogeneous systems, ranging from office applications to scientific experiments. Independently of the domain, XML documents may evolve over time, so identifying and understanding the changes they undergo becomes crucial. Some syntactic-based diff approaches have been proposed to address this problem. They are mainly designed to compare revisions of XML documents using explicit IDs to match elements. However, elements in different revisions may not share IDs due to tool incompatibility or even divergent or missing schemas. In this paper, we present Phoenix, a similarity-based approach for comparing revisions of XML documents that does not rely on explicit IDs. Phoenix uses dynamic programming and optimization algorithms to compare different features (e.g., element name, attributes, content, and sub-elements) of XML documents and calculate the similarity degree between them. We compared Phoenix with X-Diff and XyDiff, two state-of-the-art XML diff algorithms. XyDiff was the fastest approach, but failed in providing precise matching results. X-Diff presented higher efficacy in 30 of the 56 scenarios, but was slow. Phoenix performed in a fraction of the running time required by X-Diff, and achieved the best results in terms of efficacy in 26 of 56 tested scenarios. In our evaluations Phoenix was by far the most efficient approach to match elements across revisions of the same XML document.

@article{WagnerMKF18,
	author = {Stefan Wagner and Daniel M{\'{e}}ndez Fern{\'{a}}ndez and Marcos Kalinowski and Michael Felderer},
	title = {Agile Requirements Engineering in Practice: Status Quo and Critical Problems},
	journal = {CLEI Electronic Journal},
	volume = {21},
	number = {1},
	year = {2018},
	keywords = {},
	abstract = {Requirements Engineering (RE) is being treated differently in agile development when compared to more traditional development processes. Yet, there is little empirical knowledge on the state of the practice and contemporary problems in agile RE. As part of a bigger survey initiative (Naming the Pain in Requirements Engineering), the main goal of this paper is to build an empirical basis on such aspects of agile RE, based on the responses of representatives from 92 different organisations. Our survey data analyses revealed that agile RE concentrates on free-text documentation of requirements elicited with a variety of techniques. The backlog is the central means to deal with changing requirements. Commonly traces between requirements and code are explicitly managed and testing and RE are typically aligned. Furthermore, continuous improvement of RE is performed due to intrinsic motivation and RE standards are commonly practiced. Among the main problems, we highlight incomplete requirements, communication flaws and moving targets. Those problems were reported to happen commonly in agile projects and to have critical consequences, including project failure. Overall, our findings show that most organisations conduct RE in a way we would expect and that agile RE is in several aspects not so different from RE in other development processes.},
	issn = {0717-5000},	pages = {6:1--6:15},	doi = {10.19153/cleiej.21.1.6},
	urlAuthor_version = {http://www.inf.puc-rio.br/~kalinowski/publications/WagnerMKF18.pdf},
}

Requirements Engineering (RE) is being treated differently in agile development when compared to more traditional development processes. Yet, there is little empirical knowledge on the state of the practice and contemporary problems in agile RE. As part of a bigger survey initiative (Naming the Pain in Requirements Engineering), the main goal of this paper is to build an empirical basis on such aspects of agile RE, based on the responses of representatives from 92 different organisations. Our survey data analyses revealed that agile RE concentrates on free-text documentation of requirements elicited with a variety of techniques. The backlog is the central means to deal with changing requirements. Commonly traces between requirements and code are explicitly managed and testing and RE are typically aligned. Furthermore, continuous improvement of RE is performed due to intrinsic motivation and RE standards are commonly practiced. Among the main problems, we highlight incomplete requirements, communication flaws and moving targets. Those problems were reported to happen commonly in agile projects and to have critical consequences, including project failure. Overall, our findings show that most organisations conduct RE in a way we would expect and that agile RE is in several aspects not so different from RE in other development processes.

@article{WinklerKSPB18,
	author = {Dietmar Winkler and Marcos Kalinowski and Marta Sabou and Sanja Petrovic and Stefan Biffl},
	title = {Investigating a Distributed and Scalable Model Review Process},
	journal = {CLEI Electronic Journal},
	volume = {21},
	number = {1},
	year = {2018},
	keywords = {},
	abstract = {[Context] Models play an important role in Software and Systems Engineering processes. Reviews are well-established methods for model quality assurance that support early and efficient defect detection. However, traditional document-based review processes have limitations with respect to the number of experts, resources, and the document size that can be applied. [Objective] In this paper, we introduce a distributed and scalable review process for model quality assurance to (a) improve defect detection effectiveness and (b) to increase review artifact coverage. [Method] We introduce the novel concept of Expected Model Elements (EMEs) as a key concept for defect detection. EMEs can be used to drive the review process. We adapt a best-practice review process to distinguish (a) between the identification of EMEs in the reference document and (b) the use of EMEs to detect defects in the model. We design and evaluate the adapted review process with a crowdsourcing tool in a feasibility study. [Results] The study results show the feasibility of the adapted review process. Further, the study showed that inspectors using the adapted review process achieved results for defect detection effectiveness, which are comparable to the performance of inspectors using a traditional inspection process, and better defect detection efficiency. Moreover, from a practical perspective the adapted review process can be used to complement inspection efforts conducted using the traditional inspection process, enhancing the overall defect detection effectiveness. [Conclusions] Although the study shows promising results of the novel process, future investigations should consider larger and more diverse review artifacts and the effect of using limited and different scopes of artifact coverage for individual inspectors.},
	issn = {0717-5000},	pages = {4:1--4:13},	doi = {10.19153/cleiej.21.1.4},
	urlAuthor_version = {http://www.inf.puc-rio.br/~kalinowski/publications/WinklerKSPB18.pdf},
}

[Context] Models play an important role in Software and Systems Engineering processes. Reviews are well-established methods for model quality assurance that support early and efficient defect detection. However, traditional document-based review processes have limitations with respect to the number of experts, resources, and the document size that can be applied. [Objective] In this paper, we introduce a distributed and scalable review process for model quality assurance to (a) improve defect detection effectiveness and (b) to increase review artifact coverage. [Method] We introduce the novel concept of Expected Model Elements (EMEs) as a key concept for defect detection. EMEs can be used to drive the review process. We adapt a best-practice review process to distinguish (a) between the identification of EMEs in the reference document and (b) the use of EMEs to detect defects in the model. We design and evaluate the adapted review process with a crowdsourcing tool in a feasibility study. [Results] The study results show the feasibility of the adapted review process. Further, the study showed that inspectors using the adapted review process achieved results for defect detection effectiveness, which are comparable to the performance of inspectors using a traditional inspection process, and better defect detection efficiency. Moreover, from a practical perspective the adapted review process can be used to complement inspection efforts conducted using the traditional inspection process, enhancing the overall defect detection effectiveness. [Conclusions] Although the study shows promising results of the novel process, future investigations should consider larger and more diverse review artifacts and the effect of using limited and different scopes of artifact coverage for individual inspectors.

@inproceedings{MendoncaSOBLBKS18,
	author = {Mendon\c{c}a, Diogo S. and da Silva, Tarcila G. and de Oliveira, Daniel Ferreira and Brand\~{a}o, Julliany Sales and Lopes, Helio and Barbosa, Simone D. J. and Kalinowski, Marcos and von Staa, Arndt},
	title = {Applying Pattern-driven Maintenance: A Method to Prevent Latent Unhandled Exceptions in Web Applications},
	abstract = {Background: Unhandled exceptions affect the reliability of web applications. Several studies have measured the reliability of web applications in use against unhandled exceptions, showing a recurrence of the problem during the maintenance phase. Detecting latent unhandled exceptions automatically is difficult and application-specific. Hence, general approaches to deal with defects in web applications do not treat unhandled exceptions appropriately. Aims: To design and evaluate a method that can support finding, correcting, and preventing unhandled exceptions in web applications. Method: We applied the design science engineering cycle to design a method called Pattern-Driven Maintenance (PDM). PDM relies on identifying defect patterns based on application server logs and producing static analysis rules that can be used for prevention. We applied PDM to two industrial web applications involving different companies and technologies, measuring the reliability improvement and the precision of the produced static analysis rules. Results: In both cases, our approach allowed identifying defect patterns and finding latent unhandled exceptions to be fixed in the source code, enabling to completely eliminate the pattern-related failures and improving the application reliability. The static analysis rules produced by PDM achieved a precision of 59-68% in the first application and 89-100% in the second, where lessons learnt from the first evaluation were addressed. Conclusions: The results strengthen our confidence that PDM can help maintainers to improve the reliability for unhandled exceptions in other existing web applications.},
	booktitle = {Proceedings of the 12th {ACM/IEEE} International Symposium on Empirical Software Engineering and Measurement, {ESEM} 2018, Oulu, Finland, October 11-12},
	year = {2018},
	pages = {31:1--31:10},
	urlAuthor_version = {http://www.inf.puc-rio.br/~kalinowski/publications/MendoncaSOBLBKS18.pdf},
	doi = {10.1145/3239235.3268924},
} 

Background: Unhandled exceptions affect the reliability of web applications. Several studies have measured the reliability of web applications in use against unhandled exceptions, showing a recurrence of the problem during the maintenance phase. Detecting latent unhandled exceptions automatically is difficult and application-specific. Hence, general approaches to deal with defects in web applications do not treat unhandled exceptions appropriately. Aims: To design and evaluate a method that can support finding, correcting, and preventing unhandled exceptions in web applications. Method: We applied the design science engineering cycle to design a method called Pattern-Driven Maintenance (PDM). PDM relies on identifying defect patterns based on application server logs and producing static analysis rules that can be used for prevention. We applied PDM to two industrial web applications involving different companies and technologies, measuring the reliability improvement and the precision of the produced static analysis rules. Results: In both cases, our approach allowed identifying defect patterns and finding latent unhandled exceptions to be fixed in the source code, enabling to completely eliminate the pattern-related failures and improving the application reliability. The static analysis rules produced by PDM achieved a precision of 59-68% in the first application and 89-100% in the second, where lessons learnt from the first evaluation were addressed. Conclusions: The results strengthen our confidence that PDM can help maintainers to improve the reliability for unhandled exceptions in other existing web applications.

@inproceedings{MesaVVDCKL18,
  author    = {Oslien Mesa and
               Reginaldo Vieira and
               Marx L. Viana and
               Vinicius H. S. Durelli and
               Elder Cirilo and
               Marcos Kalinowski and
               Carlos Lucena},
  title     = {Understanding vulnerabilities in plugin-based web systems: an exploratory
               study of wordpress},
  abstract  = {A common software product line strategy involves plugin-based web systems that support simple and quick incorporation of custom behaviors. As a result, they have been widely adopted to create web-based applications. Indeed, the popularity of ecosystems that support plugin-based development (e.g., WordPress) is largely due to the number of customization options available as communitycontributed plugins. However, plugin-related vulnerabilities tend to be recurrent, exploitable and hard to be detected and may lead to severe consequences for the customized product. Hence, there is a need to further understand such vulnerabilities to enable preventing relevant security threats. Therefore, we conducted an exploratory study to characterize vulnerabilities caused by plugins in web-based systems. To this end, we went over WordPress vulnerability bulletins cataloged by the National Vulnerability Database as well as associated patches maintained by theWordPress plugins repository. We identified the main types of vulnerabilities caused by plugins as well as their impact and the size of the patch to fix the vulnerability. Moreover, we identified the most common security-related topics discussed among WordPress developers. We observed that, while plugin-related vulnerabilities may have severe consequences and might remain unnoticed for years before being fixed, they can commonly be mitigated with small and localized changes to the source code. The characterization helps to provide an understanding on how typical plugin-based vulnerabilities manifest themselves in practice. Such information can be helpful to steer future research on plugin-based vulnerability detection and prevention.},
  booktitle = {Proceeedings of the 22nd International Conference on Systems and Software
               Product Line - Volume 1, {SPLC} 2018, Gothenburg, Sweden, September
               10-14, 2018},
  pages     = {149--159},
  year      = {2018},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/MesaVVDCKL18.pdf},
  doi       = {10.1145/3233027.3233042},
}

A common software product line strategy involves plugin-based web systems that support simple and quick incorporation of custom behaviors. As a result, they have been widely adopted to create web-based applications. Indeed, the popularity of ecosystems that support plugin-based development (e.g., WordPress) is largely due to the number of customization options available as communitycontributed plugins. However, plugin-related vulnerabilities tend to be recurrent, exploitable and hard to be detected and may lead to severe consequences for the customized product. Hence, there is a need to further understand such vulnerabilities to enable preventing relevant security threats. Therefore, we conducted an exploratory study to characterize vulnerabilities caused by plugins in web-based systems. To this end, we went over WordPress vulnerability bulletins cataloged by the National Vulnerability Database as well as associated patches maintained by theWordPress plugins repository. We identified the main types of vulnerabilities caused by plugins as well as their impact and the size of the patch to fix the vulnerability. Moreover, we identified the most common security-related topics discussed among WordPress developers. We observed that, while plugin-related vulnerabilities may have severe consequences and might remain unnoticed for years before being fixed, they can commonly be mitigated with small and localized changes to the source code. The characterization helps to provide an understanding on how typical plugin-based vulnerabilities manifest themselves in practice. Such information can be helpful to steer future research on plugin-based vulnerability detection and prevention.

@inproceedings{LaignerKLSO18,
  author    = {Rodrigo Laigner and Marcos Kalinowski and Sergio Lifschitz and Rodrigo Salvador and Daniel de Oliveira},
  title     = {A Systematic Mapping of Software Engineering Approaches to Develop Big Data Systems},
  abstract  = {[Context] Data is being collected at an unprecedented scale. Data sets are becoming so large and complex that traditionally engineered systems may be inadequate to deal with them. While software engineering comprises a large set of approaches to support engineering robust software systems, there is no comprehensive overview of approaches that have been proposed and/or applied in the context of engineering big data systems. [Goal] This study aims at surveying existing research on big data software engineering to unveil and characterize the development approaches and major contributions. [Method] We conducted a systematic mapping study, identifying 52 related research papers, dated from 2011 to 2016. We classified and analyzed the identified approaches, their objectives, application domains, development lifecycle phase, and type of contribution. [Results] As a result, we outline the current state of the art and gaps on employing software engineering approaches to develop big data systems. For instance, we observed that the major challenges are in the area of software architecture and that more experimentation is needed to assess the classified approaches. [Conclusion] The results of this systematic mapping provide an overview on existing  approaches to support building big data systems and helps to steer future research based on the identified gaps.},
  booktitle = {44th Euromicro Conference on Software Engineering and Advanced Applications, {SEAA} 2018, Prague, Czech Republic, Aug 29-31},
  pages     = {446--453},
  year      = {2018},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/LaignerKLSO18.pdf},
  doi       = {10.1109/SEAA.2018.00079},
}

[Context] Data is being collected at an unprecedented scale. Data sets are becoming so large and complex that traditionally engineered systems may be inadequate to deal with them. While software engineering comprises a large set of approaches to support engineering robust software systems, there is no comprehensive overview of approaches that have been proposed and/or applied in the context of engineering big data systems. [Goal] This study aims at surveying existing research on big data software engineering to unveil and characterize the development approaches and major contributions. [Method] We conducted a systematic mapping study, identifying 52 related research papers, dated from 2011 to 2016. We classified and analyzed the identified approaches, their objectives, application domains, development lifecycle phase, and type of contribution. [Results] As a result, we outline the current state of the art and gaps on employing software engineering approaches to develop big data systems. For instance, we observed that the major challenges are in the area of software architecture and that more experimentation is needed to assess the classified approaches. [Conclusion] The results of this systematic mapping provide an overview on existing approaches to support building big data systems and helps to steer future research based on the identified gaps.

@inproceedings{VillamizarKVM18,
  author    = {Hugo Villamizar and Marcos Kalinowski and Marx Viana and Daniel M{\'{e}}ndez Fern{\'{a}}ndez},
  title     = {A Systematic Mapping Study on Security in Agile Requirements Engineering},
  abstract  = {[Background] The rapidly changing business environments in which many companies operate is challenging traditional Requirements Engineering (RE) approaches. This gave rise to agile approaches for RE. Security, at the same time, is an essential non-functional requirement that still tends to be difficult to address in agile development contexts. Given the fuzzy notion of “agile” in context of RE and the difficulties of appropriately handling security requirements, the overall understanding of how to handle security requirements in agile RE is still vague. [Objective] Our goal is to characterize the publication landscape of approaches that handle security requirements in agile software projects. [Method] We conducted a systematic mapping to outline relevant work and contemporary gaps for future research. [Results] In total, we identified 21 studies that met our inclusion criteria, dated from 2005 to 2017. We found that the approaches typically involve modifying agile methods, introducing new artifacts (e.g., extending the concept of user story to abuser story), or introducing guidelines to handle security issues. We also identified limitations of using these approaches related to environment, people, effort and resources. [Conclusion] Our analysis suggests that more effort needs to be invested into empirically evaluating the existing approaches and that there is an avenue for future research in the direction of mitigating the identified limitations.},
  booktitle = {44th Euromicro Conference on Software Engineering and Advanced Applications, {SEAA} 2018, Prague, Czech Republic, Aug 29-31},
  pages     = {454--461},
  year      = {2018},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/VillamizarKVM18.pdf},
  doi       = {10.1109/SEAA.2018.00080},
}

[Background] The rapidly changing business environments in which many companies operate is challenging traditional Requirements Engineering (RE) approaches. This gave rise to agile approaches for RE. Security, at the same time, is an essential non-functional requirement that still tends to be difficult to address in agile development contexts. Given the fuzzy notion of “agile” in context of RE and the difficulties of appropriately handling security requirements, the overall understanding of how to handle security requirements in agile RE is still vague. [Objective] Our goal is to characterize the publication landscape of approaches that handle security requirements in agile software projects. [Method] We conducted a systematic mapping to outline relevant work and contemporary gaps for future research. [Results] In total, we identified 21 studies that met our inclusion criteria, dated from 2005 to 2017. We found that the approaches typically involve modifying agile methods, introducing new artifacts (e.g., extending the concept of user story to abuser story), or introducing guidelines to handle security issues. We also identified limitations of using these approaches related to environment, people, effort and resources. [Conclusion] Our analysis suggests that more effort needs to be invested into empirically evaluating the existing approaches and that there is an avenue for future research in the direction of mitigating the identified limitations.

@inproceedings{SousaOOBGLKMNOLP18,
  author    = {Leonardo Da Silva Sousa and 
			   Anderson Oliveira and
			   Willian Oizumi and
			   Simone Barbosa and
			   Alessandro Garcia and 
			   Jaejoon Lee and 
			   Marcos Kalinowski and 
			   Rafael de Mello and
			   Baldoino Neto and 
			   Roberto Oliveira and 
			   Carlos Lucena and
			   Rodrigo Paes},   
  title     = {Identifying Design Problems in the Source Code: A Grounded Theory},
  abstract  = {The prevalence of design problems may cause re-engineering or even discontinuation of the system. Due to missing, informal or outdated design documentation, developers often have to rely on the source code to identify design problems. Therefore, developers have to analyze different symptoms that manifest in several
				code elements, which may quickly turn into a complex task. Although researchers have been investigating techniques to help developers in identifying design problems, there is little knowledge on how developers actually proceed to identify design problems. In order to tackle this problem, we conducted a multi-trial industrial
				experiment with professionals from 5 software companies to build a grounded theory. The resulting theory offers explanations on how developers identify design problems in practice. For instance, it reveals the characteristics of symptoms that developers consider helpful. Moreover, developers often combine different
				types of symptoms to identify a single design problem. This knowledge serves as a basis to further understand the phenomena and advance towards more effective identification techniques.},
  booktitle = {40th {IEEE/ACM} International Conference on Software Engineering, {ICSE} 2018, Gothenburg, Sweden, May 27-Jun 3},
  pages     = {921--931},
  note      = {<font color="red">ACM Distinguished Paper Award!</font>},      
  year      = {2018},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/SousaOOBGLKMNOLP18.pdf},
  doi       = {10.1145/3180155.3180239},
}

The prevalence of design problems may cause re-engineering or even discontinuation of the system. Due to missing, informal or outdated design documentation, developers often have to rely on the source code to identify design problems. Therefore, developers have to analyze different symptoms that manifest in several code elements, which may quickly turn into a complex task. Although researchers have been investigating techniques to help developers in identifying design problems, there is little knowledge on how developers actually proceed to identify design problems. In order to tackle this problem, we conducted a multi-trial industrial experiment with professionals from 5 software companies to build a grounded theory. The resulting theory offers explanations on how developers identify design problems in practice. For instance, it reveals the characteristics of symptoms that developers consider helpful. Moreover, developers often combine different types of symptoms to identify a single design problem. This knowledge serves as a basis to further understand the phenomena and advance towards more effective identification techniques.

@inproceedings{BifflKW18,
  author    = {Stefan Biffl and 
			   Marcos Kalinowski and
			   Dietmar Winkler},   
  title     = {Towards an Experiment Line on Software Inspection with Human Computation},
  abstract  = {Software inspection is an important approach to find defects in soft-ware engineering (SE) artifacts. While there has been extensive research on traditional software inspection with pen-and-paper materials, modern SE poses new environments, methods, and tools for the cooperation of software engineers. Technologies, such as Human Computation, provide tool support for distributed and tool-mediated work processes. However, there is little empirical experience on how to leverage Human Computation for software inspection. In this vision paper, we present the context for a research program on this topic and introduce the preliminary concept of a theory-based experiment line to allow designing experiment families, that can fit together to answer larger questions than individual experiments. We present an example feature model for such experiment line for Software Inspection with Human Computation and discuss its expected benefits for the research program, including the coordination of the research, design and material reuse, and aggregation facilities.},
  booktitle = {6th {IEEE/ACM} International Workshop on Conducting Empirical Studies in Industry, CESI@ICSE 2018, Gothenburg, Sweden, May 28},
  pages     = {21--24},
  year      = {2018},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/BifflKW18.pdf},
  doi       = {10.1145/3193965.3193971},
}

Software inspection is an important approach to find defects in soft-ware engineering (SE) artifacts. While there has been extensive research on traditional software inspection with pen-and-paper materials, modern SE poses new environments, methods, and tools for the cooperation of software engineers. Technologies, such as Human Computation, provide tool support for distributed and tool-mediated work processes. However, there is little empirical experience on how to leverage Human Computation for software inspection. In this vision paper, we present the context for a research program on this topic and introduce the preliminary concept of a theory-based experiment line to allow designing experiment families, that can fit together to answer larger questions than individual experiments. We present an example feature model for such experiment line for Software Inspection with Human Computation and discuss its expected benefits for the research program, including the coordination of the research, design and material reuse, and aggregation facilities.

@inproceedings{MendezTKFK18,
  author    = {Daniel M{\'{e}}ndez Fern{\'{a}}ndez and
               Michaela Tiessler and
               Marcos Kalinowski and
               Michael Felderer and
               Marco Kuhrmann},   
  title     = {On Evidence-based Risk Management in Requirements Engineering},
  abstract  = {Background: The sensitivity of Requirements Engineering (RE) to the context makes it difficult to efficiently control problems therein, thus, hampering an effective risk management devoted to allow for early corrective or even preventive measures. Problem: There is still little empirical knowledge about context-specific RE phenomena which would be necessary for an effective context-sensitive risk management in RE.
               Goal: We propose and validate an evidence-based approach to assess risks in RE using cross-company data about problems, causes and effects. Research Method: We use survey data from 228 companies and build a probabilistic network that supports the forecast of context-specific RE phenomena. We implement this approach using spreadsheets to support a light-weight risk assessment.
               Results: Our results from an initial validation in 6 companies strengthen our confidence that the approach increases the awareness for individual risk factors in RE, and the feedback further allows for disseminating our approach into practice.},
  booktitle = {Software Quality Days,
               10th International Conference, {SWQD} 2018, Vienna, Austria, January 16-19},
  pages     = {39-59},
  note      = {<font color="red">Technical Research Best Paper Award!</font>},      
  year      = {2018},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/MendezTKFK18.pdf},
  doi       = {10.1007/978-3-319-71440-0_3},
}

Background: The sensitivity of Requirements Engineering (RE) to the context makes it difficult to efficiently control problems therein, thus, hampering an effective risk management devoted to allow for early corrective or even preventive measures. Problem: There is still little empirical knowledge about context-specific RE phenomena which would be necessary for an effective context-sensitive risk management in RE. Goal: We propose and validate an evidence-based approach to assess risks in RE using cross-company data about problems, causes and effects. Research Method: We use survey data from 228 companies and build a probabilistic network that supports the forecast of context-specific RE phenomena. We implement this approach using spreadsheets to support a light-weight risk assessment. Results: Our results from an initial validation in 6 companies strengthen our confidence that the approach increases the awareness for individual risk factors in RE, and the feedback further allows for disseminating our approach into practice.

@article{MendezWKFMVC17,
  author    = {Daniel M{\'{e}}ndez Fern{\'{a}}ndez and
               Stefan Wagner and
               Marcos Kalinowski and
               Michael Felderer and
               Priscilla Mafra and
               Antonio Vetro and
               Tayana Conte and
               M.{-}T. Christiansson and
               Des Greer and
               Casper Lassenius and
               Tomi M{\"{a}}nnist{\"{o}} and
               M. Nayabi and
               Markku Oivo and
               Birgit Penzenstadler and
               Dietmar Pfahl and
               Rafael Prikladnicki and
               G{\"{u}}nther Ruhe and
               Andr{\'{e}} Schekelmann and
               Sagar Sen and
               Rodrigo O. Sp{\'{\i}}nola and
               Ahmet Tuzcu and
               Jose Luis de la Vara and
               Roel Wieringa},
  title     = {Naming the pain in requirements engineering - Contemporary problems,
               causes, and effects in practice},
  abstract  = {Requirements Engineering (RE) has received much attention in research and practice due to its importance to software project success. Its interdisciplinary nature, the dependency to the customer, and its inherent uncertainty still render the discipline difficult to investigate. This results in a lack of empirical data. These are necessary, however, to demonstrate which practically relevant RE problems exist and to what extent they matter. Motivated by this situation, we initiated the Naming the Pain in Requirements Engineering (NaPiRE) initiative which constitutes a globally distributed, bi-yearly replicated family of surveys on the status quo and problems in practical RE. In this article, we report on the qualitative analysis of data obtained from 228 companies working in 10 countries in various domains and we reveal which contemporary problems practitioners encounter. To this end, we analyse 21 problems derived from the literature with respect to their relevance and criticality in dependency to their context, and we complement this picture with a cause-effect analysis showing the causes and effects surrounding the most critical problems. Our results give us a better understanding of which problems exist and how they manifest themselves in practical environments. Thus, we provide a first step to ground contributions to RE on empirical observations which, until now, were dominated by conventional wisdom only.},
  journal   = {Empirical Software Engineering},
  issn      = {1573-7616},
  volume    = {22},
  number    = {5},
  pages     = {2298-2338},
  year      = {2017},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/MendezWKFMVC17.pdf},
  doi       = {10.1007/s10664-016-9451-7},
}

Requirements Engineering (RE) has received much attention in research and practice due to its importance to software project success. Its interdisciplinary nature, the dependency to the customer, and its inherent uncertainty still render the discipline difficult to investigate. This results in a lack of empirical data. These are necessary, however, to demonstrate which practically relevant RE problems exist and to what extent they matter. Motivated by this situation, we initiated the Naming the Pain in Requirements Engineering (NaPiRE) initiative which constitutes a globally distributed, bi-yearly replicated family of surveys on the status quo and problems in practical RE. In this article, we report on the qualitative analysis of data obtained from 228 companies working in 10 countries in various domains and we reveal which contemporary problems practitioners encounter. To this end, we analyse 21 problems derived from the literature with respect to their relevance and criticality in dependency to their context, and we complement this picture with a cause-effect analysis showing the causes and effects surrounding the most critical problems. Our results give us a better understanding of which problems exist and how they manifest themselves in practical environments. Thus, we provide a first step to ground contributions to RE on empirical observations which, until now, were dominated by conventional wisdom only.

@inproceedings{KalinowskiCPFSF17,
  author    = {Marcos Kalinowski and
               Pablo Curty and
               Aline Paes and
               Alexandre Ferreira and
               Rodrigo O. Sp{\'{\i}}nola and
               Daniel M{\'{e}}ndez Fern{\'{a}}ndez and
               Michael Felderer and
               Stefan Wagner},
  title     = {Supporting Defect Causal Analysis in Practice with Cross-Company Data
               on Causes of Requirements Engineering Problems},
  abstract  = {[Context] Defect Causal Analysis (DCA) represents an efficient practice to improve software processes. While knowledge on cause-effect relations is helpful to support DCA, collecting cause-effect data may require significant effort and time. [Goal] We propose and evaluate a new DCA approach that uses cross-company data to support the practical application of DCA. [Method] We collected cross-company data on causes of requirements engineering problems from 74 Brazilian organizations and built a Bayesian network. Our DCA approach uses the diagnostic inference of the Bayesian network to support DCA sessions. We evaluated our approach by applying a model for technology transfer to industry and conducted three consecutive evaluations: (i) in academia, (ii) with industry representatives of the Fraunhofer Project Center at UFBA, and (iii) in an industrial case study at the Brazilian National Development Bank (BNDES). [Results] We received positive feedback in all three evaluations and the cross-company data was considered helpful for determining main causes. [Conclusions] Our results strengthen our confidence in that supporting DCA with cross-company data is promising and should be further investigated. © 2017 IEEE.},			   
  booktitle = {39th {IEEE/ACM} International Conference on Software Engineering:
               Software Engineering in Practice Track, {ICSE-SEIP} 2017, Buenos Aires,
               Argentina, May 20-28, 2017},
  pages     = {223-232},
  year      = {2017},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/KalinowskiCPFSF17.pdf},
  doi       = {10.1109/ICSE-SEIP.2017.14},
}

[Context] Defect Causal Analysis (DCA) represents an efficient practice to improve software processes. While knowledge on cause-effect relations is helpful to support DCA, collecting cause-effect data may require significant effort and time. [Goal] We propose and evaluate a new DCA approach that uses cross-company data to support the practical application of DCA. [Method] We collected cross-company data on causes of requirements engineering problems from 74 Brazilian organizations and built a Bayesian network. Our DCA approach uses the diagnostic inference of the Bayesian network to support DCA sessions. We evaluated our approach by applying a model for technology transfer to industry and conducted three consecutive evaluations: (i) in academia, (ii) with industry representatives of the Fraunhofer Project Center at UFBA, and (iii) in an industrial case study at the Brazilian National Development Bank (BNDES). [Results] We received positive feedback in all three evaluations and the cross-company data was considered helpful for determining main causes. [Conclusions] Our results strengthen our confidence in that supporting DCA with cross-company data is promising and should be further investigated. © 2017 IEEE.

@inproceedings{MouraoKMMW17,
  author    = {Erica Mourao and
               Marcos Kalinowski and
               Leonardo Murta and
               Emilia Mendes and
               Claes Wohlin},
  title     = {Investigating the Use of a Hybrid Search Strategy for Systematic Reviews},
  abstract  = {[Background] Systematic Literature Reviews (SLRs) are one of the important pillars when employing an evidence-based paradigm in Software Engineering. To date most SLRs have been conducted using a search strategy involving several digital libraries. However, significant issues have been reported for digital libraries and applying such search strategy requires substantial effort. On the other hand, snowballing has recently arisen as a potentially more efficient alternative or complementary solution. Nevertheless, it requires a relevant seed set of papers. [Aims] This paper proposes and evaluates a hybrid search strategy combining searching in a specific digital library (Scopus) with backward and forward snowballing. [Method] The proposed hybrid strategy was applied to two previously published SLRs that adopted database searches. We investigate whether it is able to retrieve the same included papers with lower effort in terms of the number of analysed papers. The two selected SLRs relate respectively to elicitation techniques (not confined to Software Engineering (SE)) and to a specific SE topic on cost estimation. [Results] Our results provide preliminary support for the proposed hybrid search strategy as being suitable for SLRs investigating a specific research topic within the SE domain. Furthermore, it helps overcoming existing issues with using digital libraries in SE. [Conclusions] The hybrid search strategy provides competitive results, similar to using several digital libraries. However, further investigation is needed to evaluate the hybrid search strategy.},
  booktitle = {Proceedings of the 11th {ACM/IEEE} International Symposium on Empirical
               Software Engineering and Measurement, {ESEM} 2017, Toronto, Canada, November 9-10},
  pages     = {},
  year      = {2017},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/MouraoKMMW17.pdf},
  doi       = {10.1109/ESEM.2017.30},
}

[Background] Systematic Literature Reviews (SLRs) are one of the important pillars when employing an evidence-based paradigm in Software Engineering. To date most SLRs have been conducted using a search strategy involving several digital libraries. However, significant issues have been reported for digital libraries and applying such search strategy requires substantial effort. On the other hand, snowballing has recently arisen as a potentially more efficient alternative or complementary solution. Nevertheless, it requires a relevant seed set of papers. [Aims] This paper proposes and evaluates a hybrid search strategy combining searching in a specific digital library (Scopus) with backward and forward snowballing. [Method] The proposed hybrid strategy was applied to two previously published SLRs that adopted database searches. We investigate whether it is able to retrieve the same included papers with lower effort in terms of the number of analysed papers. The two selected SLRs relate respectively to elicitation techniques (not confined to Software Engineering (SE)) and to a specific SE topic on cost estimation. [Results] Our results provide preliminary support for the proposed hybrid search strategy as being suitable for SLRs investigating a specific research topic within the SE domain. Furthermore, it helps overcoming existing issues with using digital libraries in SE. [Conclusions] The hybrid search strategy provides competitive results, similar to using several digital libraries. However, further investigation is needed to evaluate the hybrid search strategy.

@inproceedings{WinklerSPCKB17c,
  author    = {Dietmar Winkler and
               Marta Sabou and
               Sanja Petrovic and
               Gisele Carneiro and
               Marcos Kalinowski and
               Stefan Biffl},
  title     = {Improving Model Inspection Processes with Crowdsourcing: Findings from a Controlled Experiment},
  abstract  = {The application of best-practice software inspection processes for early defect detection requires considerable human effort. Crowdsourcing approaches can support inspection activities (a) by distributing inspection effort among a group of human experts and (b) by increasing inspection control. Thus, the application of crowdsourcing techniques aims at making inspection processes more effective and efficient. In this paper, we present a crowdsourcing-supported model inspection (CSI) process and investigate its defect detection effectiveness and efficiency when inspecting an Extended Entity Relationship (EER) model. The CSI process uses so-called Expected Model Elements (EMEs) to guide CSI inspectors during defect detection. We conducted a controlled experiment on defect detection effectiveness, efficiency, and false positives. While CSI effectiveness and efficiency is lower for CSI inspectors, the number of false positives decreases. However, CSI was found promising for increasing the control of defect detection and supports the inspection of large-scale engineering models.},
  booktitle = {Systems, Software and Services Process Improvement - 24th European
               Conference, EuroSPI 2017, Ostrava, Czech Republic, September 6-8,
               2017, Proceedings},
  pages     = {125-137},
  year      = {2017},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/WinklerSPCKB17c.pdf},
  doi       = {10.1007/978-3-319-64218-5_10},
}

The application of best-practice software inspection processes for early defect detection requires considerable human effort. Crowdsourcing approaches can support inspection activities (a) by distributing inspection effort among a group of human experts and (b) by increasing inspection control. Thus, the application of crowdsourcing techniques aims at making inspection processes more effective and efficient. In this paper, we present a crowdsourcing-supported model inspection (CSI) process and investigate its defect detection effectiveness and efficiency when inspecting an Extended Entity Relationship (EER) model. The CSI process uses so-called Expected Model Elements (EMEs) to guide CSI inspectors during defect detection. We conducted a controlled experiment on defect detection effectiveness, efficiency, and false positives. While CSI effectiveness and efficiency is lower for CSI inspectors, the number of false positives decreases. However, CSI was found promising for increasing the control of defect detection and supports the inspection of large-scale engineering models.

@inproceedings{WinklerSPCKB17a,
  author    = {Dietmar Winkler and
               Marta Sabou and
               Sanja Petrovic and
               Gisele Carneiro and
               Marcos Kalinowski and
               Stefan Biffl},
  title     = {Improving Model Inspection with Crowdsourcing},
  abstract  = {Traditional Software Inspection is a well-established approach to identify defects in software artifacts and models early and efficiently. However, insufficient method and tool support hinder efficient defect detection in large software models. Recent Human Computation and Crowdsourcing processes may help to overcome this limitation by splitting complex inspection artifacts into smaller parts including a better control over defect detection tasks and increasing the scalability of inspection tasks. Therefore, we introduce a Crowdsourcing-Based Inspection (CSI) process with tool support with focus on inspection teams and the quality of defect detection. We evaluate the CSI process in a feasibility study involving 63 inspectors using the CSI process and 12 inspectors using a traditional best-practice inspection process. The CSI process was found useful by the participants. Although the preliminary results of the study were promising, the CSI process should be further investigated with typical large software engineering models.},
  booktitle = {4th {IEEE/ACM} International Workshop on CrowdSourcing in Software
               Engineering, CSI-SE@ICSE 2017, Buenos Aires, Argentina, May 22, 2017},
  pages     = {30-34},
  year      = {2017},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/WinklerSPCKB17a.pdf},
  doi       = {10.1109/CSI-SE.2017.2},
}

Traditional Software Inspection is a well-established approach to identify defects in software artifacts and models early and efficiently. However, insufficient method and tool support hinder efficient defect detection in large software models. Recent Human Computation and Crowdsourcing processes may help to overcome this limitation by splitting complex inspection artifacts into smaller parts including a better control over defect detection tasks and increasing the scalability of inspection tasks. Therefore, we introduce a Crowdsourcing-Based Inspection (CSI) process with tool support with focus on inspection teams and the quality of defect detection. We evaluate the CSI process in a feasibility study involving 63 inspectors using the CSI process and 12 inspectors using a traditional best-practice inspection process. The CSI process was found useful by the participants. Although the preliminary results of the study were promising, the CSI process should be further investigated with typical large software engineering models.

@inproceedings{WagnerFFK17,
  author    = {Stefan Wagner and Daniel M{\'{e}}ndez Fern{\'{a}}ndez and Michael Felderer and Marcos Kalinowski},
  title     = {Requirements engineering practice and problems in agile projects: Results from an international survey},
  abstract  = {Requirements engineering (RE) is considerably different in agile development than in more traditional development processes. Yet, there is little empirical knowledge on the state of the practice and contemporary problems in agile RE. As part of a bigger survey initiative (Naming the Pain in Requirements Engineering), we build an empirical basis on such aspects of agile RE. Based on the responses of representatives from 92 different organisations, we found that agile RE concentrates on free-text documentation of requirements elicited with a variety of techniques. Often, traces between requirements and code are explicitly managed and also software testing and RE are aligned. Furthermore, continuous improvement of RE is performed due to intrinsic motivation. Important experienced problems include unclear requirements and communication flaws. Overall, we found that most organisations conduct RE in a way we would expect and that agile RE is in several aspects not so different from RE in other development processes.},  
  booktitle = {CIbSE 2017 - XX Ibero-American Conference on Software Engineering},
  pages     = {85-98},
  note      = {<font color="red">Best Paper Candidate (invited for journal extension)!</font>},  
  year      = {2017},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/WagnerFFK17.pdf},
}

Requirements engineering (RE) is considerably different in agile development than in more traditional development processes. Yet, there is little empirical knowledge on the state of the practice and contemporary problems in agile RE. As part of a bigger survey initiative (Naming the Pain in Requirements Engineering), we build an empirical basis on such aspects of agile RE. Based on the responses of representatives from 92 different organisations, we found that agile RE concentrates on free-text documentation of requirements elicited with a variety of techniques. Often, traces between requirements and code are explicitly managed and also software testing and RE are aligned. Furthermore, continuous improvement of RE is performed due to intrinsic motivation. Important experienced problems include unclear requirements and communication flaws. Overall, we found that most organisations conduct RE in a way we would expect and that agile RE is in several aspects not so different from RE in other development processes.

@inproceedings{WinklerSPCKB17b,
  author    = {Winkler, D. and Sabou, M. and Petrovic, S. and Carneiro, G. and Kalinowski, M. and Biffl, S.},
  title     = {Investigating model quality assurance with a distributed and scalable review process},
  abstract  = {Models play an important role in Software and Systems Engineering processes. Reviews are well-established methods for model quality assurance that support early and efficient defect detection. However, traditional document-based review processes have limitations with respect to the number of experts, resources, and the document size that can be applied. In this paper, we introduce a distributed and scalable review process for model quality assurance to (a) improve defect detection effectiveness and (b) to increase review artifact coverage. Therefore, we introduce the novel concept of Expected Model Elements (EMEs) as a key concept for defect detection. EMEs can be used to drive the review process. We adapt a best-practice review process to distinguish (a) between the identification of EMEs in the reference document and (b) the use of EMEs to detect defects in the model. We design and evaluate the adapted review process with a crowdsourcing tool in a feasibility study. The study results show the feasibility of the adapted review process. Further, the study showed that inspectors using the adapted review process achieved results for defect detection effectiveness, which are comparable to the performance of inspectors using a traditional inspection process, and better defect detection efficiency Although the study shows promising results of the novel inspection process, future investigations should consider larger and more diverse review artifacts and the effect of the limited scope of artifact coverage for an individual inspector.},  
  booktitle = {CIbSE 2017 - XX Ibero-American Conference on Software Engineering},
  pages     = {141-154},
  note      = {<font color="red">Best Paper Candidate (invited for journal extension)!</font>},  
  year      = {2017},
  urlAuthor_version={http://www.inf.puc-rio.br/~kalinowski/publications/WinklerSPCKB17b.pdf},
}

Models play an important role in Software and Systems Engineering processes. Reviews are well-established methods for model quality assurance that support early and efficient defect detection. However, traditional document-based review processes have limitations with respect to the number of experts, resources, and the document size that can be applied. In this paper, we introduce a distributed and scalable review process for model quality assurance to (a) improve defect detection effectiveness and (b) to increase review artifact coverage. Therefore, we introduce the novel concept of Expected Model Elements (EMEs) as a key concept for defect detection. EMEs can be used to drive the review process. We adapt a best-practice review process to distinguish (a) between the identification of EMEs in the reference document and (b) the use of EMEs to detect defects in the model. We design and evaluate the adapted review process with a crowdsourcing tool in a feasibility study. The study results show the feasibility of the adapted review process. Further, the study showed that inspectors using the adapted review process achieved results for defect detection effectiveness, which are comparable to the performance of inspectors using a traditional inspection process, and better defect detection efficiency Although the study shows promising results of the novel inspection process, future investigations should consider larger and more diverse review artifacts and the effect of the limited scope of artifact coverage for an individual inspector.

@inproceedings{CarneiroLKWB17,
  author    = {Carneiro, G. and Laigner, R. and Kalinowski, M. and Winkler, D. and Biffl, S.},
  title     = {Investigating the influence of inspector learning styles on design inspections: Findings of a quasi-experiment},
  abstract  = {Software inspections are an efficient mean to improve quality. Learning Styles (LS) have been used to detect an individual's preferences to acquire and process information according to different dimensions. For requirements inspections, inspection teams with different LSs showed more effective at detecting defects than teams with similar LS. The goal of this paper is to investigate the influence of inspector's LS on the effectiveness and efficiency of design inspections. We conducted a quasi-experiment using a subset of a real requirements document and corresponding design diagrams with seeded defects, characterizing each of the participants according to their LSs. We also implemented a script to combine inspectors into nominal inspection teams. We observed that, for design inspections, teams with similar LSs were more effective and efficient than teams with dissimilar LSs. Design inspections seem to be affected more by other factors, such as defect detection techniques, than from selecting inspectors based on their learning styles. However, replications are needed to reinforce our results.},
  booktitle = {CIbSE 2017 - XX Ibero-American Conference on Software Engineering},
  pages={222-235},
  year={2017},
  urlAuthor_version={http://www.inf.puc-rio.br/~kalinowski/publications/CarneiroLKWB17.pdf},
}

Software inspections are an efficient mean to improve quality. Learning Styles (LS) have been used to detect an individual's preferences to acquire and process information according to different dimensions. For requirements inspections, inspection teams with different LSs showed more effective at detecting defects than teams with similar LS. The goal of this paper is to investigate the influence of inspector's LS on the effectiveness and efficiency of design inspections. We conducted a quasi-experiment using a subset of a real requirements document and corresponding design diagrams with seeded defects, characterizing each of the participants according to their LSs. We also implemented a script to combine inspectors into nominal inspection teams. We observed that, for design inspections, teams with similar LSs were more effective and efficient than teams with dissimilar LSs. Design inspections seem to be affected more by other factors, such as defect detection techniques, than from selecting inspectors based on their learning styles. However, replications are needed to reinforce our results.

@inproceedings{RomeiroOKO17,
  author    = {Romeiro, A. and Ocaña, K. and Kalinowski, M. and De Oliveira, D.},
  title     = {SciAgile: Aplica{\c{c}}{\~a}o de metodologias {\'a}geis em experimentos científicos baseados em simula{\c{c}}{\~a}o},
  abstract  = {Muitos dos riscos envolvidos no desenvolvimento de software tem associa{\c{c}}{\~a}o com requisitos vol{\'a}teis. Tal volatilidade motivou a cria{\c{c}}{\~a}o e a ado{\c{c}}{\~a}o de metodologias {\'a}geis, que hoje s{\~a}o uma realidade nas organiza{\c{c}}ões. A volatilidade dos requisitos tamb{\'e}m pode ser encontrada na pesquisa científica, que cada vez mais depende de complexas simula{\c{c}}ões computacionais. Tal como software comercial, o software científico, sua execu{\c{c}}{\~a}o e a an{\'a}lise dos resultados precisam ser compreensíveis, passíveis de reprodu{\c{c}}{\~a}o e livres de defeitos. Como requisitos de experimentos científicos tamb{\'e}m s{\~a}o comumente vol{\'a}teis, resolvemos investigar se as metodologias {\'a}geis podem tamb{\'e}m ser aplicadas neste contexto. Esse artigo apresenta uma abordagem, denominada SciAgile, que aplica uma s{\'e}rie de pr{\'a}ticas {\'a}geis no ciclo de vida de experimentos científicos. Para avaliar a proposta, realizamos um estudo experimental que envolveu a aplica{\c{c}}{\~a}o da abordagem SciAgile na ger{\^e}ncia de experimentos. Resultados fornecem indícios que a ado{\c{c}}{\~a}o das pr{\'a}ticas {\'a}geis propostas pode ajudar a melhorar o processo de experimenta{\c{c}}{\~a}o científica in silico.}, 
  booktitle = {CIbSE 2017 - XX Ibero-American Conference on Software Engineering},
  pages     = {15-28},
  year      = {2017},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/RomeiroOKO17.pdf},
}

Muitos dos riscos envolvidos no desenvolvimento de software tem associação com requisitos voláteis. Tal volatilidade motivou a criação e a adoção de metodologias ágeis, que hoje são uma realidade nas organizações. A volatilidade dos requisitos também pode ser encontrada na pesquisa científica, que cada vez mais depende de complexas simulações computacionais. Tal como software comercial, o software científico, sua execução e a análise dos resultados precisam ser compreensíveis, passíveis de reprodução e livres de defeitos. Como requisitos de experimentos científicos também são comumente voláteis, resolvemos investigar se as metodologias ágeis podem também ser aplicadas neste contexto. Esse artigo apresenta uma abordagem, denominada SciAgile, que aplica uma série de práticas ágeis no ciclo de vida de experimentos científicos. Para avaliar a proposta, realizamos um estudo experimental que envolveu a aplicação da abordagem SciAgile na gerência de experimentos. Resultados fornecem indícios que a adoção das práticas ágeis propostas pode ajudar a melhorar o processo de experimentação científica in silico.

@ARTICLE{FariasSKMS17,
  author    = {de Freitas Farias, M.A. and Santos, J.A. and Kalinowski, M. and Mendon{\c{c}}a, M. and Spínola, R.O.},
  title     = {Investigating the identification of technical debt through code comment analysis},
  abstract  = {In order to effectively manage technical debt (TD), a set of indicators has been used by automated approaches to identify TD items. However, some debt items may not be directly identified using only metrics collected from the source code. CVM-TD is a model to support the identification of technical debt by considering the developer point of view when identifying TD through code comment analysis. In this paper, we investigate the use of CVM-TD with the purpose of characterizing factors that affect the accuracy of the identification of TD, and the most chosen patterns by participants as decisive to indicate TD items. We performed a controlled experiment investigating the accuracy of CVM-TD and the influence of English skills and developer experience factors. We also investigated if the contextualized vocabulary provided by CVM-TD points to candidate comments that are considered indicators of technical debt by participants. The results indicated that CVM-TD provided promising results considering the accuracy values. English reading skills have an impact on the TD detection process. We could not conclude that the experience level affects this process. We identified a list of the 20 most chosen patterns by participants as decisive to indicate TD items. The results motivate us continuing to explore code comments in the context of TD identification process in order to improve CVM-TD. © Springer International Publishing AG 2017.},
  journal   = {Lecture Notes in Business Information Processing (ICEIS 2016 - Revised Selected Papers)},
  volume    = {291},
  pages     = {284-309},
  year      = {2017},  
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/FariasSKMS17.pdf},  
  doi       = {10.1007/978-3-319-62386-3_14},
}

In order to effectively manage technical debt (TD), a set of indicators has been used by automated approaches to identify TD items. However, some debt items may not be directly identified using only metrics collected from the source code. CVM-TD is a model to support the identification of technical debt by considering the developer point of view when identifying TD through code comment analysis. In this paper, we investigate the use of CVM-TD with the purpose of characterizing factors that affect the accuracy of the identification of TD, and the most chosen patterns by participants as decisive to indicate TD items. We performed a controlled experiment investigating the accuracy of CVM-TD and the influence of English skills and developer experience factors. We also investigated if the contextualized vocabulary provided by CVM-TD points to candidate comments that are considered indicators of technical debt by participants. The results indicated that CVM-TD provided promising results considering the accuracy values. English reading skills have an impact on the TD detection process. We could not conclude that the experience level affects this process. We identified a list of the 20 most chosen patterns by participants as decisive to indicate TD items. The results motivate us continuing to explore code comments in the context of TD identification process in order to improve CVM-TD. © Springer International Publishing AG 2017.

@inproceedings{TitoEMOK17,
  author    = {Lucas Tito and
               Alexandre Estebanez and
               Andr{\'{e}}a Magalh{\~{a}}es Magdaleno and
               Daniel de Oliveira and
               Marcos Kalinowski},
  title     = {A Systematic Mapping of Software Requirements Negotiation Techniques},
  abstract  = {[Context] Eliciting requirements is a commonly discussed task. However, after they are ready, it is essentially important for a software project that these requirements are sufficient for stakeholders to reach their goals. Therefore, techniques to negotiate schedule, price, quality, and scope among stakeholders are important. [Goal] This paper aims at identifying and presenting characteristics of techniques that have been proposed and/or used to negotiate software requirements. [Method] A mapping study was planned and conducted to identify techniques and to capture their characteristics. Those characteristics include description, environment (e.g. academic, industrial), the types of research being published, and the types of primary studies. The main findings of the papers, and the advantages and disadvantages reported for theses techniques were also summarized. [Results] We mapped the characteristics of 10 different requirements negotiation techniques identified in 33 papers which met our inclusion criteria. We found that most of the identified techniques can be seen as variations of the seminal WinWin requirements negotiation technique proposed in 1994. [Conclusions] The conducted mapping study provides an interesting overview of the area and may also be useful to ground future research on this topic. ©2017 by SCITEPRESS - Science and Technology Publications, Lda. All rights reserved.},
  booktitle = {{ICEIS} 2017 - Proceedings of the 19th International Conference on
               Enterprise Information Systems, Volume 2, Porto, Portugal, April 26-29,
               2017},
  pages     = {518-525},
  year      = {2017},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/TitoEMOK17.pdf},
  doi       = {10.5220/0006362605180525},
}

[Context] Eliciting requirements is a commonly discussed task. However, after they are ready, it is essentially important for a software project that these requirements are sufficient for stakeholders to reach their goals. Therefore, techniques to negotiate schedule, price, quality, and scope among stakeholders are important. [Goal] This paper aims at identifying and presenting characteristics of techniques that have been proposed and/or used to negotiate software requirements. [Method] A mapping study was planned and conducted to identify techniques and to capture their characteristics. Those characteristics include description, environment (e.g. academic, industrial), the types of research being published, and the types of primary studies. The main findings of the papers, and the advantages and disadvantages reported for theses techniques were also summarized. [Results] We mapped the characteristics of 10 different requirements negotiation techniques identified in 33 papers which met our inclusion criteria. We found that most of the identified techniques can be seen as variations of the seminal WinWin requirements negotiation technique proposed in 1994. [Conclusions] The conducted mapping study provides an interesting overview of the area and may also be useful to ground future research on this topic. ©2017 by SCITEPRESS - Science and Technology Publications, Lda. All rights reserved.

@inproceedings{BatistaMK17,
  author    = {Matheus Froes Batista and
               Andrea Magalhaes Magdaleno and
               Marcos Kalinowski},
  title     = {A Survey on the use of Social BPM in Practice in Brazilian Organizations},
  abstract  = {Social BPM is the combination of Business Process Management (BPM) with social and collaborative techniques for the purpose of exploring collaboration among stakeholders throughout the BPM lifecycle. Its goals are to reduce common problems in BPM by ensuring collaboration and transparency. To the best of our knowledge, there is no information on how Social BPM is being used in organizational environments and on its impacts. This study aims at showing how Brazilian organizations are using Social BPM practices and technologies. Therefore, considering time and cost constraints, a survey was conducted with employees from different companies in order to collect data on their usage of BPM collaborative practices. The survey received 31 replies and 3 of the respondents were also interviewed in order to provide depth to their answers and to enhance the overall understanding. The results show that collaboration happens predominantly in design, modeling, and improvement phases. Collaboration still happens mainly without formal planning and without tool support.},
  booktitle = {{SBSI} 2017 - Proceedings of the Brazilian Symposium on Information Systems, Lavras, Brazil},
  pages     = {1-10},
  year      = {2017},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/BatistaMK17.pdf},
  doi       = {},
}

Social BPM is the combination of Business Process Management (BPM) with social and collaborative techniques for the purpose of exploring collaboration among stakeholders throughout the BPM lifecycle. Its goals are to reduce common problems in BPM by ensuring collaboration and transparency. To the best of our knowledge, there is no information on how Social BPM is being used in organizational environments and on its impacts. This study aims at showing how Brazilian organizations are using Social BPM practices and technologies. Therefore, considering time and cost constraints, a survey was conducted with employees from different companies in order to collect data on their usage of BPM collaborative practices. The survey received 31 replies and 3 of the respondents were also interviewed in order to provide depth to their answers and to enhance the overall understanding. The results show that collaboration happens predominantly in design, modeling, and improvement phases. Collaboration still happens mainly without formal planning and without tool support.

@inproceedings{FelizardoMKSV16,
  author    = {Katia Romero Felizardo and
               Emilia Mendes and
               Marcos Kalinowski and
               {\'{E}}rica Ferreira de Souza and
               Nandamudi L. Vijaykumar},
  title     = {Using Forward Snowballing to update Systematic Reviews in Software
               Engineering},
  abstract  = {Background: A Systematic Literature Review (SLR) is a methodology used to aggregate relevant evidence related to one or more research questions. Whenever new evidence is published after the completion of a SLR, this SLR should be updated in order to preserve its value. However, updating SLRs involves significant effort. Objective: The goal of this paper is to investigate the application of forward snowballing to support the update of SLRs. Method: We compare outcomes of an update achieved using the forward snowballing versus a published update using the search-based approach, i.e., searching for studies in electronic databases using a search string. Results: Forward snowballing showed a higher precision and a slightly lower recall. It reduced in more than five times the number of primary studies to filter however missed one relevant study. Conclusions: Due to its high precision, we believe that the use of forward snowballing considerably reduces the effort in updating SLRs in Software Engineering; however the risk of missing relevant papers should not be underrated. © 2016 ACM.},			   
  booktitle = {Proceedings of the 10th {ACM/IEEE} International Symposium on Empirical
               Software Engineering and Measurement, {ESEM} 2016, Ciudad Real, Spain,
               September 8-9, 2016},
  pages     = {53:1-53:6},
  note      = {<font color="red">Best Short Paper Award!</font>},  
  year      = {2016},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/FelizardoMKSV16.pdf},
  doi       = {10.1145/2961111.2962630},
}

Background: A Systematic Literature Review (SLR) is a methodology used to aggregate relevant evidence related to one or more research questions. Whenever new evidence is published after the completion of a SLR, this SLR should be updated in order to preserve its value. However, updating SLRs involves significant effort. Objective: The goal of this paper is to investigate the application of forward snowballing to support the update of SLRs. Method: We compare outcomes of an update achieved using the forward snowballing versus a published update using the search-based approach, i.e., searching for studies in electronic databases using a search string. Results: Forward snowballing showed a higher precision and a slightly lower recall. It reduced in more than five times the number of primary studies to filter however missed one relevant study. Conclusions: Due to its high precision, we believe that the use of forward snowballing considerably reduces the effort in updating SLRs in Software Engineering; however the risk of missing relevant papers should not be underrated. © 2016 ACM.

@inproceedings{MafraKFFW16,
  author    = {Priscilla Mafra and
               Marcos Kalinowski and
               Daniel M{\'{e}}ndez Fern{\'{a}}ndez and
               Michael Felderer and
               Stefan Wagner},
  title     = {Towards Guidelines for Preventing Critical Requirements Engineering
               Problems},
  abstract  = {[Context] Problems in Requirements Engineering (RE) can lead to serious consequences during the software development lifecycle. [Goal] The goal of this paper is to propose empirically-based guidelines that can be used by different types of organisations according to their size (small, medium or large) and process model (agile or plan-driven) to help them in preventing such problems. [Method] We analysed data from a survey on RE problems answered by 228 organisations in 10 different countries. [Results] We identified the most critical RE problems, their causes and mitigation actions, organizing this information by clusters of size and process model. Finally, we analysed the causes and mitigation actions of the critical problems of each cluster to get further insights into how to prevent them. [Conclusions] Based on our results, we suggest preliminary guidelines for preventing critical RE problems in response to context characteristics of the companies. © 2016 IEEE.},			   
  booktitle = {42th Euromicro Conference on Software Engineering and Advanced Applications,
               {SEAA} 2016, Limassol, Cyprus, August 31 - Sept. 2, 2016},
  pages     = {25-29},
  year      = {2016},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/MafraKFFW16.pdf},
  doi       = {10.1109/SEAA.2016.50},
}

[Context] Problems in Requirements Engineering (RE) can lead to serious consequences during the software development lifecycle. [Goal] The goal of this paper is to propose empirically-based guidelines that can be used by different types of organisations according to their size (small, medium or large) and process model (agile or plan-driven) to help them in preventing such problems. [Method] We analysed data from a survey on RE problems answered by 228 organisations in 10 different countries. [Results] We identified the most critical RE problems, their causes and mitigation actions, organizing this information by clusters of size and process model. Finally, we analysed the causes and mitigation actions of the critical problems of each cluster to get further insights into how to prevent them. [Conclusions] Based on our results, we suggest preliminary guidelines for preventing critical RE problems in response to context characteristics of the companies. © 2016 IEEE.

@inproceedings{MendesFMSKS16,
  author    = {Thiago Souto Mendes and
               M{\'{a}}rio Andr{\'{e}} de Freitas Farias and
               Manoel G. Mendon{\c{c}}a and
               Henrique Frota Soares and
               Marcos Kalinowski and
               Rodrigo Oliveira Sp{\'{\i}}nola},
  title     = {Impacts of agile requirements documentation debt on software projects:
               a retrospective study},
  abstract  = {Documentation debt is a type of technical debt that describes problems in documentation such as missing, inadequate or incomplete artifacts. Unlike traditional methods, agile methodologies usually employ short iterative cycles and rely on tacit knowledge within a team. In particular, Agile Requirements (AR) (e.g., user stories) tend to reduce the focus on requirements specification activities. This scenario contributes to the occurrence of documentation debt. The goal of this paper is to investigate the impact that this type of debt brings to projects developed by using AR. We address this goal by performing a retrospective study in a real software project that used AR in its development. Our analysis was concentrated on data from 132 maintenance and evolution tasks. Of this total, 65 were related to the presence of documentation debt and were performed within a timeframe of 18 months. The findings indicated an extra maintenance effort of about 47% of the total effort estimated for developing the project and an extra cost of about 48% of the initial cost of the development phase. © 2016 ACM.},			   
  booktitle = {Proceedings of the 31st Annual {ACM} Symposium on Applied Computing,
               Pisa, Italy, April 4-8, 2016},
  pages     = {1290-1295},
  year      = {2016},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/MendesFMSKS16.pdf},
  doi       = {10.1145/2851613.2851761},
}

Documentation debt is a type of technical debt that describes problems in documentation such as missing, inadequate or incomplete artifacts. Unlike traditional methods, agile methodologies usually employ short iterative cycles and rely on tacit knowledge within a team. In particular, Agile Requirements (AR) (e.g., user stories) tend to reduce the focus on requirements specification activities. This scenario contributes to the occurrence of documentation debt. The goal of this paper is to investigate the impact that this type of debt brings to projects developed by using AR. We address this goal by performing a retrospective study in a real software project that used AR in its development. Our analysis was concentrated on data from 132 maintenance and evolution tasks. Of this total, 65 were related to the presence of documentation debt and were performed within a timeframe of 18 months. The findings indicated an extra maintenance effort of about 47% of the total effort estimated for developing the project and an extra cost of about 48% of the initial cost of the development phase. © 2016 ACM.

@inproceedings{KalinowskiFCSPW16,
  author    = {Marcos Kalinowski and
               Michael Felderer and
               Tayana Conte and
               Rodrigo O. Sp{\'{\i}}nola and
               Rafael Prikladnicki and
               Dietmar Winkler and
               Daniel M{\'{e}}ndez Fern{\'{a}}ndez and
               Stefan Wagner},
  title     = {Preventing Incomplete/Hidden Requirements: Reflections on Survey Data
               from Austria and Brazil},
  abstract  = {[Context] Many software projects fail due to problems in requirements engineering (RE). [Goal] The goal of this paper is analyzing a specific and relevant RE problem in detail: incomplete/hidden requirements. [Method] We replicated a global family of RE surveys with representatives of software organizations in Austria and Brazil. We used the data to (a) characterize the criticality of the selected RE problem, and to (b) analyze the reported main causes and mitigation actions. Based on the analysis, we discuss how to prevent the problem. [Results] The survey includes 14 different organizations in Austria and 74 in Brazil, including small, medium and large sized companies, conducting both, plan-driven and agile development processes. Respondents from both countries cited the incomplete/hidden requirements problem as one of the most critical RE problems. We identified and graphically represented the main causes and documented solution options to address these causes. Further, we compiled a list of reported mitigation actions. [Conclusions] From a practical point of view, this paper provides further insights into common causes of incomplete/hidden requirements and on how to prevent this problem. © Springer International Publishing Switzerland 2016.},			   
  booktitle = {Software Quality. The Future of Systems- and Software Development,
               8th International Conference, {SWQD} 2016, Vienna, Austria, January
               18-21, 2016, Proceedings},
  pages     = {63-78},
  note      = {<font color="red">Technical Research Best Paper Award!</font>},    
  year      = {2016},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/KalinowskiFCSPW16.pdf},
  doi       = {10.1007/978-3-319-27033-3_5},
}

[Context] Many software projects fail due to problems in requirements engineering (RE). [Goal] The goal of this paper is analyzing a specific and relevant RE problem in detail: incomplete/hidden requirements. [Method] We replicated a global family of RE surveys with representatives of software organizations in Austria and Brazil. We used the data to (a) characterize the criticality of the selected RE problem, and to (b) analyze the reported main causes and mitigation actions. Based on the analysis, we discuss how to prevent the problem. [Results] The survey includes 14 different organizations in Austria and 74 in Brazil, including small, medium and large sized companies, conducting both, plan-driven and agile development processes. Respondents from both countries cited the incomplete/hidden requirements problem as one of the most critical RE problems. We identified and graphically represented the main causes and documented solution options to address these causes. Further, we compiled a list of reported mitigation actions. [Conclusions] From a practical point of view, this paper provides further insights into common causes of incomplete/hidden requirements and on how to prevent this problem. © Springer International Publishing Switzerland 2016.

@inproceedings{OliveiraEGMPKL16,
  author    = {Roberto Oliveira and
               Bernardo Jos{\'{e}} da Silva Est{\'{a}}cio and
               Alessandro F. Garcia and
               Sabrina Marczak and
               Rafael Prikladnicki and
               Marcos Kalinowski and
               Carlos Jos{\'{e}} Pereira de Lucena},
  title     = {Identifying Code Smells with Collaborative Practices: {A} Controlled
               Experiment},
  abstract  = {Code smells are often considered as key indicators of software quality degradation. If code smells are not systematically removed from a program, its continuous degradation may lead to either major maintenance effort or the complete redesign of the system. For several reasons, software developers introduce smells in their code as soon as they start to learn programming. If novice developers are ought to become either proficient programmers or skilled code reviewers, they should be early prepared to effectively identify code smells in existing programs. However, effective identification of code smells is often not a non-trivial task in particular to a novice developer working in isolation. Thus, the use of collaborative practices may have the potential to support developers in improving their effectiveness on this task at their early stages of their careers. These practices offer the opportunity for two or more developers analyzing the source code together and collaboratively reason about potential smells prevailing on it. Pair Programming (PP) and Coding Dojo Randori (CDR) are two increasingly adopted practices for improving the effectiveness of developers with limited or no knowledge in software engineering tasks, including code review tasks. However, there is no broad understanding about the impact of these collaborative practices on the effectiveness of code smell identification. This paper presents a controlled experiment involving 28 novice developers, aimed at assessing the effectiveness of collaborative practices in the identification of code smells. We compared PP and CDR with solo programming in order to better distinguish their impact on the effective identification of code smells. Our study is also the first in the literature to observe how novice developers work individually and together to identify smells. Our results suggest that collaborative practices contribute to the effectiveness on the identification of a wide range of code smells. Our findings can also be used in practice to guide educators, researchers or teams on improving detection and training on code smell identification. © 2016 IEEE.},			   
  booktitle = {2016 {X} Brazilian Symposium on Software Components, Architectures
               and Reuse, {SBCARS} 2016, Maring{\'{a}}, Brazil, September 19-20,
               2016},
  pages     = {61-70},
  year      = {2016},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/OliveiraEGMPKL16.pdf},
  doi       = {10.1109/SBCARS.2016.18},
}

Code smells are often considered as key indicators of software quality degradation. If code smells are not systematically removed from a program, its continuous degradation may lead to either major maintenance effort or the complete redesign of the system. For several reasons, software developers introduce smells in their code as soon as they start to learn programming. If novice developers are ought to become either proficient programmers or skilled code reviewers, they should be early prepared to effectively identify code smells in existing programs. However, effective identification of code smells is often not a non-trivial task in particular to a novice developer working in isolation. Thus, the use of collaborative practices may have the potential to support developers in improving their effectiveness on this task at their early stages of their careers. These practices offer the opportunity for two or more developers analyzing the source code together and collaboratively reason about potential smells prevailing on it. Pair Programming (PP) and Coding Dojo Randori (CDR) are two increasingly adopted practices for improving the effectiveness of developers with limited or no knowledge in software engineering tasks, including code review tasks. However, there is no broad understanding about the impact of these collaborative practices on the effectiveness of code smell identification. This paper presents a controlled experiment involving 28 novice developers, aimed at assessing the effectiveness of collaborative practices in the identification of code smells. We compared PP and CDR with solo programming in order to better distinguish their impact on the effective identification of code smells. Our study is also the first in the literature to observe how novice developers work individually and together to identify smells. Our results suggest that collaborative practices contribute to the effectiveness on the identification of a wide range of code smells. Our findings can also be used in practice to guide educators, researchers or teams on improving detection and training on code smell identification. © 2016 IEEE.

@inproceedings{FariasSSKMS16,
  author    = {M{\'{a}}rio Andr{\'{e}} de Freitas Farias and
               Jos{\'{e}} Amancio M. Santos and
               Andr{\'{e}} Batista da Silva and
               Marcos Kalinowski and
               Manoel G. Mendon{\c{c}}a and
               Rodrigo Oliveira Sp{\'{\i}}nola},
  title     = {Investigating the Use of a Contextualized Vocabulary in the Identification
               of Technical Debt: {A} Controlled Experiment},
  abstract  = {In order to effectively manage technical debt (TD), a set of indicators has been used by automated approaches to identify TD items. However, some debt may not be directly identified using only metrics collected from the source code. CVM-TD is a model to support the identification of technical debt by considering the developer point of view when identifying TD through code comment analysis. In this paper, we analyze the use of CVM-TD with the purpose of characterizing factors that affect the accuracy of the identification of TD. We performed a controlled experiment investigating the accuracy of CVM-TD and the influence of English skills and developer experience factors. The results indicated that CVM-TD provided promising results considering the accuracy values. English reading skills have an impact on the TD detection process. We could not conclude that the experience level affects this process. Finally, we also observed that many comments suggested by CVM-TD were considered good indicators of TD. The results motivate us continuing to explore code comments in the context of TD identification process in order to improve CVM-TD. Copyright © 2016 by SCITEPRESS - Science and Technology Publications, Lda. All rights reserved.},			   
  booktitle = {{ICEIS} 2016 - Proceedings of the 18th International Conference on
               Enterprise Information Systems, Volume 1, Rome, Italy, April 25-28,
               2016},
  pages     = {369-378},
  note      = {<font color="red">Best Paper Award!</font>},    
  year      = {2016},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/FariasSSKMS16.pdf},
  doi       = {10.5220/0005914503690378},
}

In order to effectively manage technical debt (TD), a set of indicators has been used by automated approaches to identify TD items. However, some debt may not be directly identified using only metrics collected from the source code. CVM-TD is a model to support the identification of technical debt by considering the developer point of view when identifying TD through code comment analysis. In this paper, we analyze the use of CVM-TD with the purpose of characterizing factors that affect the accuracy of the identification of TD. We performed a controlled experiment investigating the accuracy of CVM-TD and the influence of English skills and developer experience factors. The results indicated that CVM-TD provided promising results considering the accuracy values. English reading skills have an impact on the TD detection process. We could not conclude that the experience level affects this process. Finally, we also observed that many comments suggested by CVM-TD were considered good indicators of TD. The results motivate us continuing to explore code comments in the context of TD identification process in order to improve CVM-TD. Copyright © 2016 by SCITEPRESS - Science and Technology Publications, Lda. All rights reserved.

@inproceedings{LupoKR16,
  author    = {Peter Lupo and
               Marcos Kalinowski and
               Ana Regina Rocha},
  title     = {Infraestrutura para um Corpo de Conhecimento em Melhoria de Processos de Software Baseado no MR-MPS-SW},
  abstract  = {Software development organizations commonly follow maturity models to guide their software process improvement initiatives, being responsible for identifying and choosing the technologies to address these models’ requirements. This paper proposes an infrastructure to organize a body of knowledge of technologies that support the implementation of the MR-MPS-SW focused on the software process improvement professionals’ needs.},
  booktitle = {{SBQS} 2016 - Proceedings of the Brazilian Symposium on Software Quality, Maceio, Brazil},
  pages     = {151-165},
  year      = {2016},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/LupoKR16.pdf},
  doi       = {},
}

Software development organizations commonly follow maturity models to guide their software process improvement initiatives, being responsible for identifying and choosing the technologies to address these models’ requirements. This paper proposes an infrastructure to organize a body of knowledge of technologies that support the implementation of the MR-MPS-SW focused on the software process improvement professionals’ needs.

@inproceedings{GracioliVK16,
  author    = {Carlos Gracioli and
               Jos{\'e} Viterbo and
               Marcos Kalinowski},
  title     = {Um Levantamento do Uso de T{\'e}cnicas de Aprendizado de M{\'a}quina na Estimativa de Esfor{\c{c}}o de Software},
  abstract  = {A estimativa de esfor{\c{c}}o {\'e} uma pr{\'a}tica essencial no processo de desenvolvimento de software, sendo necess{\'a}ria para que o prazo e os custos do projeto sejam previstos antecipadamente e o projeto n{\~a}o fracasse. Este artigo apresenta os resultados de um levantamento que identificou 10 artigos que estudam o uso de t{\'e}cnicas de aprendizado de m{\'a}quina nas estimativas de esfor{\c{c}}o, para definir quais permitem alcan{\c{c}}ar boas aproxima{\c{c}}ões em rela{\c{c}}{\~a}o ao esfor{\c{c}}o real. Diversas t{\'e}cnicas foram pesquisadas e as mais utilizadas foram as supervisionadas de regress{\~a}o. As particularidades dos estudos dificultam a compara{\c{c}}{\~a}o dos resultados, mas as redes neurais e bayesianas obtiveram bons resultados nas avalia{\c{c}}ões analisadas, ou seja, uma menor taxa de erros nas estimativas. A vari{\'a}vel de entrada mais relevante para as diferentes t{\'e}cnicas, como esperado, {\'e} o tamanho estimado do projeto.},
  booktitle = {{Workpedia} 2016},
  pages     = {1-10},
  year      = {2016},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/GracioliVK16.pdf},
  doi       = {},
}

A estimativa de esforço é uma prática essencial no processo de desenvolvimento de software, sendo necessária para que o prazo e os custos do projeto sejam previstos antecipadamente e o projeto não fracasse. Este artigo apresenta os resultados de um levantamento que identificou 10 artigos que estudam o uso de técnicas de aprendizado de máquina nas estimativas de esforço, para definir quais permitem alcançar boas aproximações em relação ao esforço real. Diversas técnicas foram pesquisadas e as mais utilizadas foram as supervisionadas de regressão. As particularidades dos estudos dificultam a comparação dos resultados, mas as redes neurais e bayesianas obtiveram bons resultados nas avaliações analisadas, ou seja, uma menor taxa de erros nas estimativas. A variável de entrada mais relevante para as diferentes técnicas, como esperado, é o tamanho estimado do projeto.

@article{FernandezWKSTCSP15,
  author    = {Daniel M{\'{e}}ndez Fern{\'{a}}ndez and
               Stefan Wagner and
               Marcos Kalinowski and
               Andr{\'{e}} Schekelmann and
               Ahmet Tuzcu and
               Tayana Conte and
               Rodrigo O. Sp{\'{\i}}nola and
               Rafael Prikladnicki},
  title     = {Naming the Pain in Requirements Engineering: Comparing Practices in
               Brazil and Germany},
  abstract  = {As part of the Naming the Pain in Requirements Engineering (NaPiRE) initiative, researchers compared problems that companies in Brazil and Germany encountered during requirements engineering (RE). The key takeaway was that in RE, human interaction is necessary for eliciting and specifying high-quality requirements, regardless of country, project type, or company size. © 2015 IEEE.},			   
  journal   = {{IEEE} Software},
  volume    = {32},
  number    = {5},
  pages     = {16-23},
  year      = {2015},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/FernandezWKSTCSP15.pdf},
  doi       = {10.1109/MS.2015.122},
}

As part of the Naming the Pain in Requirements Engineering (NaPiRE) initiative, researchers compared problems that companies in Brazil and Germany encountered during requirements engineering (RE). The key takeaway was that in RE, human interaction is necessary for eliciting and specifying high-quality requirements, regardless of country, project type, or company size. © 2015 IEEE.

@article{KalinowskiWSFDT15,
  author    = {Marcos Kalinowski and
               Kival Chaves Weber and
               Gleison Santos and
               Nelson Franco and
               Virginia Duarte and
               Guilherme Travassos},
  title     = {Software Process Improvement Results in Brazil Based on the MPS-SW Model},
  abstract  = {In December 2003, the MPS.BR Program was launched aimed at establishing and disseminating a software process reference model - the MPS-SW - allowing micro, small, and medium-sized enterprises and large organizations to achieve the benefits of process improvement. Today, 12 years later, the achieved results exceed MPS.BR’s pre-defined benchmarks in several ways. This article aims to provide an overview on the MPS-SW model and present the results, describing its nationwide adoption in Brazil (more than 600 assessments spread across the country) and outcomes of two recent surveys concerning the impact of such adoption in the software industry. The authors planned surveys to capture the impact from two different and complementary points of view: the qualitative perception of the customers (sponsors of MPS-SW adopting organizations) and the performance results of the organizations that adopted the model (for example, concerning productivity, quality, and estimation accuracy). Results of the qualitative survey indicate that the adoption was motivated by both business and technical reasons and that most sponsors are satisfied with the obtained improvements and would recommend the MPS-SW model. Results of the survey on performance results indicate higher productivity, quality, and estimation accuracy for organizations assessed in higher maturity levels.},
  journal   = {Software Quality Professional Journal},
  volume    = {17},
  number    = {4},
  pages     = {14-28},
  year      = {2015},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/KalinowskiWSFDT15.pdf},
  doi       = {},
}

In December 2003, the MPS.BR Program was launched aimed at establishing and disseminating a software process reference model - the MPS-SW - allowing micro, small, and medium-sized enterprises and large organizations to achieve the benefits of process improvement. Today, 12 years later, the achieved results exceed MPS.BR’s pre-defined benchmarks in several ways. This article aims to provide an overview on the MPS-SW model and present the results, describing its nationwide adoption in Brazil (more than 600 assessments spread across the country) and outcomes of two recent surveys concerning the impact of such adoption in the software industry. The authors planned surveys to capture the impact from two different and complementary points of view: the qualitative perception of the customers (sponsors of MPS-SW adopting organizations) and the performance results of the organizations that adopted the model (for example, concerning productivity, quality, and estimation accuracy). Results of the qualitative survey indicate that the adoption was motivated by both business and technical reasons and that most sponsors are satisfied with the obtained improvements and would recommend the MPS-SW model. Results of the survey on performance results indicate higher productivity, quality, and estimation accuracy for organizations assessed in higher maturity levels.

@inproceedings{EstacioOMKGPL15,
  author    = {Bernardo Jos{\'{e}} da Silva Est{\'{a}}cio and
               Roberto Oliveira and
               Sabrina Marczak and
               Marcos Kalinowski and
               Alessandro Garcia and
               Rafael Prikladnicki and
               Carlos Lucena},
  title     = {Evaluating Collaborative Practices in Acquiring Programming Skills:
               Findings of a Controlled Experiment},
  abstract  = {[Context] Collaborative programming is achieved when two or more programmers develop software together. Pair Programming and Coding Dojo Randori are two increasingly adopted practices for collaborative programming. While the former encourages the collaboration in pairs, the latter promotes collaboration in groups. However, there is no broad understanding about the impact of these practices on the acquisition of programming skills. [Goal] In this study, we empirically compare the influence of both collaborative practices on two essential aspects of skill acquisition: motivation and learning. [Method] We conducted a controlled experiment with novice programmers applying solo programming and both collaborative practices to three different programming exercises using a crossed design. [Results] Our results showed that, while both practices outperformed solo programming, they also presented complementary benefits on acquiring programming skills. For instance, the programmers inserted less code anomalies in Coding Dojo Randori sessions than in Pair Programming sessions. On the other hand, the motivation was often considered to be stronger in the latter than in the former. [Conclusions] Our results suggest that the use of collaborative practices is particularly promising for acquiring programming skills, when programmers have little or no practical experience with software development. © 2015 IEEE.},			   
  booktitle = {29th Brazilian Symposium on Software Engineering, {SBES} 2015, Belo
               Horizonte, MG, Brazil, September 21-26, 2015},
  pages     = {150-159},
  year      = {2015},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/EstacioOMKGPL15.pdf},
  doi       = {10.1109/SBES.2015.24},
}

[Context] Collaborative programming is achieved when two or more programmers develop software together. Pair Programming and Coding Dojo Randori are two increasingly adopted practices for collaborative programming. While the former encourages the collaboration in pairs, the latter promotes collaboration in groups. However, there is no broad understanding about the impact of these practices on the acquisition of programming skills. [Goal] In this study, we empirically compare the influence of both collaborative practices on two essential aspects of skill acquisition: motivation and learning. [Method] We conducted a controlled experiment with novice programmers applying solo programming and both collaborative practices to three different programming exercises using a crossed design. [Results] Our results showed that, while both practices outperformed solo programming, they also presented complementary benefits on acquiring programming skills. For instance, the programmers inserted less code anomalies in Coding Dojo Randori sessions than in Pair Programming sessions. On the other hand, the motivation was often considered to be stronger in the latter than in the former. [Conclusions] Our results suggest that the use of collaborative practices is particularly promising for acquiring programming skills, when programmers have little or no practical experience with software development. © 2015 IEEE.

@inproceedings{KalinowskiSCPFW15,
  author    = {Marcos Kalinowski and
               Rodrigo O. Sp{\'{\i}}nola and
               Tayana Conte and
               Rafael Prikladnicki and
               Daniel M{\'{e}}ndez Fern{\'{a}}ndez and
               Stefan Wagner},
  title     = {Towards Building Knowledge on Causes of Critical Requirements Engineering
               Problems},
  abstract  = {[Context] Many software projects fail due to problems in requirements engineering (RE). [Objective] The goal of this paper is to gather information on relevant RE problems and to represent knowledge on their most common causes. [Method] We replicated a global family of RE surveys in Brazil and used the data to identify critical RE problems and to build probabilistic causeeffect diagrams to represent knowledge on their common causes. [Results] The survey was answered by 74 different organizations, including small, medium and very large sized companies, conducting both, plan-driven and agile development. The most critical RE problems, according to those organizations, are related to communication and to incomplete or underspecified requirements. We provide the full probabilistic cause-effect diagrams with knowledge on common causes of the most critical identified RE problems online. [Conclusion] We believe that the knowledge presented in the diagrams can be helpful to support organizations in conducting causal analysis sessions by providing an initial understanding on what usually causes critical RE problems. Copyright © 2015 by KSI Research Inc. and Knowledge Systems Institute Graduate School.},
  booktitle = {The 27th International Conference on Software Engineering and Knowledge
               Engineering, {SEKE} 2015, Wyndham Pittsburgh University Center, Pittsburgh,
               PA, USA, July 6-8, 2015},
  pages     = {1-6},
  year      = {2015},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/KalinowskiSCPFW15.pdf},
  doi       = {10.18293/SEKE2015-220},
}

[Context] Many software projects fail due to problems in requirements engineering (RE). [Objective] The goal of this paper is to gather information on relevant RE problems and to represent knowledge on their most common causes. [Method] We replicated a global family of RE surveys in Brazil and used the data to identify critical RE problems and to build probabilistic causeeffect diagrams to represent knowledge on their common causes. [Results] The survey was answered by 74 different organizations, including small, medium and very large sized companies, conducting both, plan-driven and agile development. The most critical RE problems, according to those organizations, are related to communication and to incomplete or underspecified requirements. We provide the full probabilistic cause-effect diagrams with knowledge on common causes of the most critical identified RE problems online. [Conclusion] We believe that the knowledge presented in the diagrams can be helpful to support organizations in conducting causal analysis sessions by providing an initial understanding on what usually causes critical RE problems. Copyright © 2015 by KSI Research Inc. and Knowledge Systems Institute Graduate School.

@article{BifflKREW14,
  author    = {Stefan Biffl and
               Marcos Kalinowski and
               Rick Rabiser and
               Fajar J. Ekaputra and
               Dietmar Winkler},
  title     = {Systematic Knowledge Engineering: Building Bodies of Knowledge from
               Published Research},
  abstract  = {Context. Software engineering researchers conduct systematic literature reviews (SLRs) to build bodies of knowledge (BoKs). Unfortunately, relevant knowledge collected in the SLR process is not publicly available, which considerably slows down building BoKs incrementally. Objective. We present and evaluate the Systematic Knowledge Engineering (SKE) process to support efficiently building BoKs from published research. Method. SKE is based on the SLR process and on Knowledge Engineering practices to build a Knowledge Base (KB) by reusing intermediate data extraction results from SLRs. We evaluated the feasibility of applying SKE by building a Software Inspection BoK KB from published experiments and a Software Product Line BoK KB from published experience reports. We compared the effort, benefits, and risks of building BoK KBs regarding the SKE and the traditional SLR processes. Results. The application of SKE for incrementally collecting and organizing knowledge in the context of a BoK was feasible for different domains and different types of evidence. While the efforts for conducting the SKE and traditional SLR processes are comparable, SKE provides significant benefits for building BoKs. Conclusions. SKE enables researchers in a scientific community to reuse and incrementally build knowledge in a BoK. SKE is ready to be evaluated in other software engineering domains. © 2014 World Scientific Publishing Company.},
  journal   = {International Journal of Software Engineering and Knowledge Engineering},
  volume    = {24},
  number    = {10},
  pages     = {1533-1572},
  year      = {2014},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/BifflKREW14.pdf},
  doi       = {10.1142/S021819401440018X},
}

Context. Software engineering researchers conduct systematic literature reviews (SLRs) to build bodies of knowledge (BoKs). Unfortunately, relevant knowledge collected in the SLR process is not publicly available, which considerably slows down building BoKs incrementally. Objective. We present and evaluate the Systematic Knowledge Engineering (SKE) process to support efficiently building BoKs from published research. Method. SKE is based on the SLR process and on Knowledge Engineering practices to build a Knowledge Base (KB) by reusing intermediate data extraction results from SLRs. We evaluated the feasibility of applying SKE by building a Software Inspection BoK KB from published experiments and a Software Product Line BoK KB from published experience reports. We compared the effort, benefits, and risks of building BoK KBs regarding the SKE and the traditional SLR processes. Results. The application of SKE for incrementally collecting and organizing knowledge in the context of a BoK was feasible for different domains and different types of evidence. While the efforts for conducting the SKE and traditional SLR processes are comparable, SKE provides significant benefits for building BoKs. Conclusions. SKE enables researchers in a scientific community to reuse and incrementally build knowledge in a BoK. SKE is ready to be evaluated in other software engineering domains. © 2014 World Scientific Publishing Company.

@article{KalinowskiBSR14,
  author    = {Marcos Kalinowski and
               Stefan Biffl and
               Rodrigo Oliveira Sp{\'{\i}}nola and
               Sheila S. Reinehr},
  title     = {From project-oriented to service-oriented software development: an
               industrial experience guided by a service reference model},
  abstract  = {[Background] In organizations with software systems in production, new and often unexpected requirements for development come up due to strategic, tactical, and operational customer needs. In this context, it is a strategic advantage for software suppliers to be able to provide software services that meet these demands faster and with less overhead than negotiating traditional value-neutral project-oriented software deliveries. [Case description] This article reports on the industrial experience of restructuring the supplier-side software development process into a value-based service-oriented format, guided by a service reference model. A service level agreement (SLA) was established between supplier and customer reflecting the business needs and values. The report describes the contractual aspects and internal managerial controls employed to facilitate the compliance of the provided services with the SLA, including the integrated use of a managerial spreadsheet, an issue-tracking system, and a Kanban chart. [Discussion and evaluation] The feasibility and results of restructuring software development into a service-oriented format are evaluated. Major results were that only moderate effort was required, around one person month, due to the support of the service reference model and a sufficient level of previously installed capabilities, and that the goals regarding improved quality, productivity, and customer satisfaction were successfully achieved. Additionally we discuss stakeholder needs, the support from the service reference model, the lessons learned, and the success factors for such restructuring. [Conclusions] Restructuring software development in the format of continuous service delivery, guided by a service reference model, is feasible and for suitable contexts can provide significant benefits concerning quality, productivity, and customer satisfaction.},			   
  journal   = {Journal of Software Engineering Research and Development},
  volume    = {2},
  pages     = {1-21},
  year      = {2014},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/KalinowskiBSR14.pdf},
  doi       = {10.1186/s40411-014-0010-x},
}

[Background] In organizations with software systems in production, new and often unexpected requirements for development come up due to strategic, tactical, and operational customer needs. In this context, it is a strategic advantage for software suppliers to be able to provide software services that meet these demands faster and with less overhead than negotiating traditional value-neutral project-oriented software deliveries. [Case description] This article reports on the industrial experience of restructuring the supplier-side software development process into a value-based service-oriented format, guided by a service reference model. A service level agreement (SLA) was established between supplier and customer reflecting the business needs and values. The report describes the contractual aspects and internal managerial controls employed to facilitate the compliance of the provided services with the SLA, including the integrated use of a managerial spreadsheet, an issue-tracking system, and a Kanban chart. [Discussion and evaluation] The feasibility and results of restructuring software development into a service-oriented format are evaluated. Major results were that only moderate effort was required, around one person month, due to the support of the service reference model and a sufficient level of previously installed capabilities, and that the goals regarding improved quality, productivity, and customer satisfaction were successfully achieved. Additionally we discuss stakeholder needs, the support from the service reference model, the lessons learned, and the success factors for such restructuring. [Conclusions] Restructuring software development in the format of continuous service delivery, guided by a service reference model, is feasible and for suitable contexts can provide significant benefits concerning quality, productivity, and customer satisfaction.

@inproceedings{MendesKMFS14,
  author    = {Emilia Mendes and
               Marcos Kalinowski and
               Daves Martins and
               Filomena Ferrucci and
               Federica Sarro},
  title     = {Cross- vs. within-company cost estimation studies revisited: an extended
               systematic review},
  abstract  = {[Objective] The objective of this paper is to extend a previously conducted systematic literature review (SLR) that investigated under what circumstances individual organizations would be able to rely on cross-company based estimation models. [Method] We applied the same methodology used in the SLR we are extending herein (covering the period 2006-2013) based on primary studies that compared predictions from cross-company models with predictions from within-company models constructed from analysis of project data. [Results] We identified 11 additional papers; however two of these did not present independent results and one had inconclusive findings. Two of the remaining eight papers presented both, trials where cross-company predictions were not significantly different from within-company predictions and others where they were significantly different. Four found that cross-company models gave prediction accuracy significantly different from within-company models (one of them in favor of cross-company models), while two found no significant difference. The main pattern when examining the study related factors was that studies where cross-company predictions were significantly different from within-company predictions employed larger within-company data sets. [Conclusions] Overall, half of the analyzed evidence indicated that cross-company estimation models are not significantly worse than within-company estimation models. Moreover, there is some evidence that sample size does not imply in higher estimation accuracy, and that samples for building estimation models should be carefully selected/filtered based on quality control and project similarity aspects. The results need to be combined with the findings from the SLR we are extending to allow further investigating this topic. Copyright 2014 ACM.},			   
  booktitle = {18th International Conference on Evaluation and Assessment in Software
               Engineering, {EASE} '14, London, England, United Kingdom, May 13-14,
               2014},
  pages     = {12:1-12:10},
  year      = {2014},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/MendesKMFS14.pdf},
  doi       = {10.1145/2601248.2601284},
}

[Objective] The objective of this paper is to extend a previously conducted systematic literature review (SLR) that investigated under what circumstances individual organizations would be able to rely on cross-company based estimation models. [Method] We applied the same methodology used in the SLR we are extending herein (covering the period 2006-2013) based on primary studies that compared predictions from cross-company models with predictions from within-company models constructed from analysis of project data. [Results] We identified 11 additional papers; however two of these did not present independent results and one had inconclusive findings. Two of the remaining eight papers presented both, trials where cross-company predictions were not significantly different from within-company predictions and others where they were significantly different. Four found that cross-company models gave prediction accuracy significantly different from within-company models (one of them in favor of cross-company models), while two found no significant difference. The main pattern when examining the study related factors was that studies where cross-company predictions were significantly different from within-company predictions employed larger within-company data sets. [Conclusions] Overall, half of the analyzed evidence indicated that cross-company estimation models are not significantly worse than within-company estimation models. Moreover, there is some evidence that sample size does not imply in higher estimation accuracy, and that samples for building estimation models should be carefully selected/filtered based on quality control and project similarity aspects. The results need to be combined with the findings from the SLR we are extending to allow further investigating this topic. Copyright 2014 ACM.

@inproceedings{BifflKENCW14,
  author    = {Stefan Biffl and
               Marcos Kalinowski and
               Fajar J. Ekaputra and
               Amadeu Anderlin-Neto and
               Tayana Conte and
               Dietmar Winkler},
  title     = {Towards a semantic knowledge base on threats to validity and control
               actions in controlled experiments},
  abstract  = {[Context] Experiment planners need to be aware of relevant Threats to Validity (TTVs), so they can devise effective control actions or accept the risk. [Objective] The aim of this paper is to introduce a TTV knowledge base (KB) that supports experiment planners in identifying relevant TTVs in their research context and actions to control these TTVs. [Method] We identified requirements, designed and populated a TTV KB with data extracted during a systematic review: 63 TTVs and 149 control actions from 206 peer-reviewed published software engineering experiments. We conducted an initial proof of concept on the feasibility of using the TTV KB and analyzed its content. [Results] The proof of concept and content analysis provided indications that experiment planners can benefit from an extensible TTV KB for identifying relevant TTVs and control actions in their specific context. [Conclusions] The TTV KB should be further evaluated and evolved in a variety of software engineering contexts. Copyright 2014 ACM.},			   
  booktitle = {2014 {ACM-IEEE} International Symposium on Empirical Software Engineering
               and Measurement, {ESEM} '14, Torino, Italy, September 18-19, 2014},
  pages     = {49:1-49:4},
  year      = {2014},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/BifflKENCW14.pdf},
  doi       = {10.1145/2652524.2652568},
}

[Context] Experiment planners need to be aware of relevant Threats to Validity (TTVs), so they can devise effective control actions or accept the risk. [Objective] The aim of this paper is to introduce a TTV knowledge base (KB) that supports experiment planners in identifying relevant TTVs in their research context and actions to control these TTVs. [Method] We identified requirements, designed and populated a TTV KB with data extracted during a systematic review: 63 TTVs and 149 control actions from 206 peer-reviewed published software engineering experiments. We conducted an initial proof of concept on the feasibility of using the TTV KB and analyzed its content. [Results] The proof of concept and content analysis provided indications that experiment planners can benefit from an extensible TTV KB for identifying relevant TTVs and control actions in their specific context. [Conclusions] The TTV KB should be further evaluated and evolved in a variety of software engineering contexts. Copyright 2014 ACM.

@inproceedings{KovalenkoWKSB14,
  author    = {Olga Kovalenko and
               Dietmar Winkler and
               Marcos Kalinowski and
               Estefan{\'{\i}}a Serral and
               Stefan Biffl},
  title     = {Engineering Process Improvement in Heterogeneous Multi-disciplinary
               Environments with Defect Causal Analysis},
  abstract  = {Multi-disciplinary engineering environments, e.g., in automation systems engineering, typically involve different stakeholder groups and engineering disciplines using a variety of specific tools and data models. Defects in individual disciplines can have a major impact on product and process quality in terms of additional cost and effort for defect repair and can lead to project delays. Early defects detection and avoidance in future projects are key challenges for project and quality managers to improve the product and process quality. In this paper we present an adaptation of the defect causal analysis (DCA) approach, which has been found effective and efficient to improve product quality in software engineering contexts. Applying DCA in multi-disciplinary engineering environments enables a systematic analysis of defects and candidate root causes, and can help providing countermeasures for product and process quality. The feasibility study of the adapted DCA has shown that the adaptation is useful and enables improving defect detection and prevention in multi-disciplinary engineering projects and fosters engineering process improvement. © Springer-Verlag Berlin Heidelberg 2014.},			   
  booktitle = {Systems, Software and Services Process Improvement - 21st European
               Conference, EuroSPI 2014, Luxembourg, June 25-27, 2014. Proceedings},
  pages     = {73-85},
  year      = {2014},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/KovalenkoWKSB14.pdf},
  doi       = {10.1007/978-3-662-43896-1_7},
}

Multi-disciplinary engineering environments, e.g., in automation systems engineering, typically involve different stakeholder groups and engineering disciplines using a variety of specific tools and data models. Defects in individual disciplines can have a major impact on product and process quality in terms of additional cost and effort for defect repair and can lead to project delays. Early defects detection and avoidance in future projects are key challenges for project and quality managers to improve the product and process quality. In this paper we present an adaptation of the defect causal analysis (DCA) approach, which has been found effective and efficient to improve product quality in software engineering contexts. Applying DCA in multi-disciplinary engineering environments enables a systematic analysis of defects and candidate root causes, and can help providing countermeasures for product and process quality. The feasibility study of the adapted DCA has shown that the adaptation is useful and enables improving defect detection and prevention in multi-disciplinary engineering projects and fosters engineering process improvement. © Springer-Verlag Berlin Heidelberg 2014.

@inproceedings{RiveroKC14,
  author    = {Luis Rivero and
               Marcos Kalinowski and
               Tayana Conte},
  title     = {Practical Findings from Applying Innovative Design Usability Evaluation
               Technologies for Mockups of Web Applications},
  abstract  = {Webflow is an optimal perceived user experience which improves the user orientation and navigational behaviors towards Web applications. In order for Web applications to achieve Webflow in terms of navigation, it is necessary to enhance their degree of user interaction. However, despite their importance, user interaction features that can improve ease of use and navigation are mostly evaluated late in development. This paper presents practical findings from applying proposed Design Usability Evaluation (DUE) technologies. Such technologies allow the usability inspection of Web applications mockups to reduce the cost of identifying and fixing navigation and ease of use problems early in the development process. We planned an empirical study to evaluate: (a) the capability of applying these technologies without training, and (b) the perceived ease of use and utility of the technologies, using the Technology Acceptance Model. The results show that the DUE technologies could be used without training to find problems affecting navigation and ease of use of Web applications. Also, practitioners agreed that using the DUE technologies could enhance their performance during an inspection. Finally, we provide a list of some of the identified user interaction problems that, if avoided, could aid in the improvement of Webflow. © 2014 IEEE.},			   
  booktitle = {47th Hawaii International Conference on System Sciences, {HICSS} 2014,
               Waikoloa, HI, USA, January 6-9, 2014},
  pages     = {3054-3063},
  year      = {2014},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/RiveroKC14.pdf},
  doi       = {10.1109/HICSS.2014.380},
}

Webflow is an optimal perceived user experience which improves the user orientation and navigational behaviors towards Web applications. In order for Web applications to achieve Webflow in terms of navigation, it is necessary to enhance their degree of user interaction. However, despite their importance, user interaction features that can improve ease of use and navigation are mostly evaluated late in development. This paper presents practical findings from applying proposed Design Usability Evaluation (DUE) technologies. Such technologies allow the usability inspection of Web applications mockups to reduce the cost of identifying and fixing navigation and ease of use problems early in the development process. We planned an empirical study to evaluate: (a) the capability of applying these technologies without training, and (b) the perceived ease of use and utility of the technologies, using the Technology Acceptance Model. The results show that the DUE technologies could be used without training to find problems affecting navigation and ease of use of Web applications. Also, practitioners agreed that using the DUE technologies could enhance their performance during an inspection. Finally, we provide a list of some of the identified user interaction problems that, if avoided, could aid in the improvement of Webflow. © 2014 IEEE.

@inproceedings{KalinowskiWFBDZS14,
  author    = {Marcos Kalinowski and
               Kival Chaves Weber and
               Nelson Franco and
               Elidiane Barroso and
               Virginia Duarte and
               David Zanetti and
               Gleison Santos},
  title     = {Results of 10 Years of Software Process Improvement in Brazil Based
               on the {MPS-SW} Model},
  abstract  = {[Context] In December 2003 the MPS. BR Program was launched aiming at establishing and disseminating a software process reference model - the MPS-SW - allowing both micro, small and medium-sized enterprises and large organizations to achieve the benefits of process improvement. Nowadays, ten years later, the achieved results exceed MPS. BR's predefined benchmarks in several ways. [Objective] This paper aims at providing an overview on the MPS-SW model and presenting these results, describing its nationwide adoption in Brazil (more than 500 assessments spread across the country) and outcomes of two recent surveys concerning the impact of such adoption in the software industry. [Method] We planned surveys to capture the impact from two different and complementary points of view: the qualitative perception of the customers (sponsors of MPS-SW adopting organizations) and the performance results of organizations that adopted the model (e.g. concerning productivity, quality and estimation accuracy). [Results] Results of the qualitative survey indicated that the adoption was motivated by both business and technical reasons and that most sponsors are satisfied with the obtained improvements and would recommend the MPS-SW model. Results of the survey on performance results indicated higher productivity, quality and estimation accuracy for organizations assessed in higher maturity levels. © 2014 IEEE.},
  booktitle = {9th International Conference on the Quality of Information and Communications
               Technology, {QUATIC} 2014, Guimaraes, Portugal, September 23-26, 2014},
  note      = {<font color="red">Best Paper Candidate (invited for journal extension)!</font>},  			   			   
  pages     = {28-37},
  year      = {2014},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/KalinowskiWFBDZS14.pdf},
  doi       = {10.1109/QUATIC.2014.11},
}

[Context] In December 2003 the MPS. BR Program was launched aiming at establishing and disseminating a software process reference model - the MPS-SW - allowing both micro, small and medium-sized enterprises and large organizations to achieve the benefits of process improvement. Nowadays, ten years later, the achieved results exceed MPS. BR's predefined benchmarks in several ways. [Objective] This paper aims at providing an overview on the MPS-SW model and presenting these results, describing its nationwide adoption in Brazil (more than 500 assessments spread across the country) and outcomes of two recent surveys concerning the impact of such adoption in the software industry. [Method] We planned surveys to capture the impact from two different and complementary points of view: the qualitative perception of the customers (sponsors of MPS-SW adopting organizations) and the performance results of organizations that adopted the model (e.g. concerning productivity, quality and estimation accuracy). [Results] Results of the qualitative survey indicated that the adoption was motivated by both business and technical reasons and that most sponsors are satisfied with the obtained improvements and would recommend the MPS-SW model. Results of the survey on performance results indicated higher productivity, quality and estimation accuracy for organizations assessed in higher maturity levels. © 2014 IEEE.

@inproceedings{BifflKESW14,
  author    = {Stefan Biffl and
               Marcos Kalinowski and
               Fajar J. Ekaputra and
               Estefan{\'{\i}}a Serral and
               Dietmar Winkler},
  title     = {Building Empirical Software Engineering Bodies of Knowledge with Systematic
               Knowledge Engineering},
  abstract  = {[Context] Empirical software engineering (EMSE) researchers conduct systematic literature reviews (SLRs) to build bodies of knowledge (BoKs). Unfortunately, valuable knowledge collected in the SLR process is publicly available only to a limited extent, which considerably slows down building BoKs incrementally. [Objective] In this paper, we introduce the Systematic Knowledge Engineering (SKE) process to support building up BoKs from empirical studies efficiently. [Method] SKE is based on the SLR process and on Knowledge Engineering (KE) practices to provide a Knowledge Base (KB) with semantic technologies that enable reusing intermediate data extraction results and querying of empirical evidence. We evaluated SKE by building a software inspection EMSE BoK KB from knowledge acquired by controlled experiments. We elicited relevant queries from EMSE researchers and systematically integrated information from 30 representative research papers into the KB. [Results] The resulting KB was effective in answering the queries, enabling knowledge reuse for analyses beyond the results from the SLR process. [Conclusion] SKE showed promising results in the software inspection context and should be evaluated in other contexts for building EMSE BoKs faster. Copyright © 2014 by Knowledge Systems Institute Graduate School.},			   
  booktitle = {The 26th International Conference on Software Engineering and Knowledge
               Engineering, Hyatt Regency, Vancouver, BC, Canada, July 1-3, 2013.},
  note      = {<font color="red">Best Paper Candidate (invited for journal extension)!</font>},  			   
  pages     = {552-559},
  year      = {2014},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/BifflKESW14.pdf}
}

[Context] Empirical software engineering (EMSE) researchers conduct systematic literature reviews (SLRs) to build bodies of knowledge (BoKs). Unfortunately, valuable knowledge collected in the SLR process is publicly available only to a limited extent, which considerably slows down building BoKs incrementally. [Objective] In this paper, we introduce the Systematic Knowledge Engineering (SKE) process to support building up BoKs from empirical studies efficiently. [Method] SKE is based on the SLR process and on Knowledge Engineering (KE) practices to provide a Knowledge Base (KB) with semantic technologies that enable reusing intermediate data extraction results and querying of empirical evidence. We evaluated SKE by building a software inspection EMSE BoK KB from knowledge acquired by controlled experiments. We elicited relevant queries from EMSE researchers and systematically integrated information from 30 representative research papers into the KB. [Results] The resulting KB was effective in answering the queries, enabling knowledge reuse for analyses beyond the results from the SLR process. [Conclusion] SKE showed promising results in the software inspection context and should be evaluated in other contexts for building EMSE BoKs faster. Copyright © 2014 by Knowledge Systems Institute Graduate School.

@inproceedings{KalinowskiMT14,
  author    = {Marcos Kalinowski and
               Emilia Mendes and
               Guilherme Horta Travassos},
  title     = {An Industry Ready Defect Causal Analysis Approach Exploring Bayesian
               Networks},
  abstract  = {Defect causal analysis (DCA) has shown itself an efficient means to improve the quality of software processes and products. A DCA approach exploring Bayesian networks, called DPPI (Defect Prevention-Based Process Improvement), resulted from research following an experimental strategy. Its conceptual phase considered evidence-based guidelines acquired through systematic reviews and feedback from experts in the field. Afterwards, in order to move towards industry readiness the approach evolved based on results of an initial proof of concept and a set of primary studies. This paper describes the experimental strategy followed and provides an overview of the resulting DPPI approach. Moreover, it presents results from applying DPPI in industry in the context of a real software development lifecycle, which allowed further comprehension and insights into using the approach from an industrial perspective. © Springer International Publishing Switzerland 2014.},			   
  booktitle = {Software Quality. Model-Based Approaches for Advanced Software and
               Systems Engineering - 6th International Conference, {SWQD} 2014, Vienna,
               Austria, January 14-16, 2014. Proceedings},
  pages     = {12-33},
  note      = {<font color="red">Industrial Experience Best Paper Award!</font>},    
  year      = {2014},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/KalinowskiMT14.pdf},
  doi       = {10.1007/978-3-319-03602-1_2},
}

Defect causal analysis (DCA) has shown itself an efficient means to improve the quality of software processes and products. A DCA approach exploring Bayesian networks, called DPPI (Defect Prevention-Based Process Improvement), resulted from research following an experimental strategy. Its conceptual phase considered evidence-based guidelines acquired through systematic reviews and feedback from experts in the field. Afterwards, in order to move towards industry readiness the approach evolved based on results of an initial proof of concept and a set of primary studies. This paper describes the experimental strategy followed and provides an overview of the resulting DPPI approach. Moreover, it presents results from applying DPPI in industry in the context of a real software development lifecycle, which allowed further comprehension and insights into using the approach from an industrial perspective. © Springer International Publishing Switzerland 2014.

@book{TravassosK14a,
  author    = {Guilherme Horta Travassos and
               Marcos Kalinowski}, 
  title     = {iMPS 2013 – Evid{\^e}ncias Sobre o Desempenho das Empresas que Adotaram o Modelo MPS-SW [PORTUGUESE VERSION]},
  publisher = {Softex},
  year      = {2014},
  note      = {ISBN: 978-85-99334-75-1, 102 p.},
  isbn      = {978-85-99334-75-1},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/TravassosK14a.pdf}
}

@book{TravassosK14b,
  author    = {Guilherme Horta Travassos and
               Marcos Kalinowski}, 
  title     = {iMPS 2013 – Evidencias Sobre el Desempeño de las Empresas que Adoptaron el Modelo MPS-SW [SPANISH VERSION]},
  publisher = {Softex},
  year      = {2014},
  note      = {ISBN: 978-85-99334-75-1, 97 p.},
  isbn      = {978-85-99334-75-1},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/TravassosK14b.pdf},
}

@techreport{BifflKEMW14,
     title = {Towards Efficient Support for Theory Identification from Published Experimental Research Reports},
     author = {Stefan Biffl and Marcos Kalinowski and Fajar J. Ekaputra and Guilherme G. Martins and Dietmar Winkler},
     group = {},
     year = {2014},
     institution = {Vienna University of Technology, Institute of Software Technology and Interactive Systems},
     pages = {1-10},	 
     urlAuthor_version = {http://www.inf.puc-rio.br/~kalinowski/publications/BifflKEMW14.pdf},	 
}

@inproceedings{KalinowskiR13,
  author    = {Marcos Kalinowski and
               Sheila Reinehr},
  title     = {Estruturando Desenvolvimento de Software como um Servi{\c{c}}o de TI: Uma Experi{\^e}ncia Pr{\'a}tica},
  abstract  = {In organizational environments involving systems in production, it is not always possible to predict development demands that will arise. In this scenario, it becomes a business advantage for suppliers to structure themselves in order to provide services able to deal with these demands, properly handling their volume and their treatment priority. In this paper, we report the experience of structuring software development as an IT service delivery according to a service level agreement (SLA) established along with the customer to meet their business needs. Both, the external vision, through contractual terms, and the internal vision, through managerial controls used to meet the SLA, are presented.},
  booktitle = {{SBQS} 2013 - Proceedings of the Brazilian Symposium on Software Quality, Salvador, Brazil},
  pages     = {318-325},
  note      = {<font color="red">Best Paper Candidate (invited for journal extension)!</font>},    
  year      = {2013},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/KalinowskiR13.pdf},
  doi       = {},
}

In organizational environments involving systems in production, it is not always possible to predict development demands that will arise. In this scenario, it becomes a business advantage for suppliers to structure themselves in order to provide services able to deal with these demands, properly handling their volume and their treatment priority. In this paper, we report the experience of structuring software development as an IT service delivery according to a service level agreement (SLA) established along with the customer to meet their business needs. Both, the external vision, through contractual terms, and the internal vision, through managerial controls used to meet the SLA, are presented.

@inproceedings{JordaoK13,
  author    = {Livia Jordao and
               Marcos Kalinowski},
  title     = {Investigando a Aplicabilidade do MPS-SV na Melhoria de Servi{\c{c}}os de Desenvolvimento e Manuten{\c{c}}{\~a}o de Software},
  abstract  = {The main goal of the MPS.BR program is to develop and disseminate reference models that meet the needs of the Brazilian Software and IT Services Industry. Currently, the family of models consists of the MPS-SW for Software and the MPS-SV for IT Services. Given that the MPS-SW has been largely adopted and the MPS-SV was recently published, this paper describes a survey conducted to investigate the applicability of the MPS-SV in organizations that already adopt the MPS-SW and that seek to improve their software development and maintenance services. The results indicate that, from the point of view of MPS-SW implementation consultants, implementing MPS-SV in this context is facilitated by the organizational culture already established in the context of the MPS-SW and may bring benefits in terms of productivity and quality.},
  booktitle = {{WAMPS} 2013 - Workshop Anual do MPS.BR, Campinas, Brazil},
  pages     = {152-163},
  year      = {2013},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/JordaoK13.pdf},
  doi       = {},
}

The main goal of the MPS.BR program is to develop and disseminate reference models that meet the needs of the Brazilian Software and IT Services Industry. Currently, the family of models consists of the MPS-SW for Software and the MPS-SV for IT Services. Given that the MPS-SW has been largely adopted and the MPS-SV was recently published, this paper describes a survey conducted to investigate the applicability of the MPS-SV in organizations that already adopt the MPS-SW and that seek to improve their software development and maintenance services. The results indicate that, from the point of view of MPS-SW implementation consultants, implementing MPS-SV in this context is facilitated by the organizational culture already established in the context of the MPS-SW and may bring benefits in terms of productivity and quality.

@article{SilvaSK13,
  author    = {Edmilson Rodrigues da Silva and
               Rodrigo Oliveira Sp{\'{\i}}nola and
               Marcos Kalinowski},
  title     = {Implementando ger{\^e}ncia de reutiliza{\c{c}}{\~a}o de software},
  abstract  = {Este artigo apresenta uma abordagem de ger{\^e}ncia de reutiliza{\c{c}}{\~a}o para uma empresa real, seguindo os preceitos do processo de Ger{\^e}ncia de Reutiliza{\c{c}}{\~a}o do modelo de refer{\^e}ncia do MPS.BR, definindo o que vem a ser um ativo reutiliz{\'a}vel no contexto de sistemas de gerenciamento integradoproduzidos por esta empresa e propondo um mecanismo de armazenamento para controle desses ativos reutiliz{\'a}veis. Um ativo reutiliz{\'a}vel, na abordagem do processo de ger{\^e}ncia de reutiliza{\c{c}}{\~a}o apresentada neste artigo, {\'e} um conjunto de arquivos (documenta{\c{c}}{\~a}o, implementa{\c{c}}{\~a}o, etc) derivado da realiza{\c{c}}{\~a}o de um caso de uso de um sistema de gerenciamento integrado. Uma prova de conceito mostrou a viabilidade de aplicar da abordagem apresentada.},  
  journal   = {Engenharia de Software Magazine},
  volume    = {59},
  number    = {},
  pages     = {31-39},
  year      = {2013},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/SilvaSK13.pdf},
  doi       = {},
}

Este artigo apresenta uma abordagem de gerência de reutilização para uma empresa real, seguindo os preceitos do processo de Gerência de Reutilização do modelo de referência do MPS.BR, definindo o que vem a ser um ativo reutilizável no contexto de sistemas de gerenciamento integradoproduzidos por esta empresa e propondo um mecanismo de armazenamento para controle desses ativos reutilizáveis. Um ativo reutilizável, na abordagem do processo de gerência de reutilização apresentada neste artigo, é um conjunto de arquivos (documentação, implementação, etc) derivado da realização de um caso de uso de um sistema de gerenciamento integrado. Uma prova de conceito mostrou a viabilidade de aplicar da abordagem apresentada.

@inproceedings{SilvaK13,
  author    = {Edmilson Rodrigues da Silva and
               Marcos Kalinowski},
  title     = {Uma Abordagem para Ger{\^e}ncia de Reutiliza{\c{c}}{\~a}o de Software no Contexto da Constru{\c{c}}{\~a}o de Sistemas de Gerenciamento Integrado},
  abstract  = {Reutiliza{\c{c}}{\~a}o de software {\'e} o processo de cria{\c{c}}{\~a}o de sistemas de software a partir de software existente ao inv{\'e}s de construí-los a partir do zero (KRUEGER, 1992). Sabe-se que a reutiliza{\c{c}}{\~a}o pode trazer benefícios para uma organiza{\c{c}}{\~a}o, como o aumento da qualidade e da produtividade e a redu{\c{c}}{\~a}o dos custos e do tempo de desenvolvimento de sistemas. Neste contexto, a ger{\^e}ncia de reutiliza{\c{c}}{\~a}o tem o propósito de gerenciar o ciclo de vida de ativos reutiliz{\'a}veis, propondo um mecanismo de armazenamento para busca e recupera{\c{c}}{\~a}o dos mesmos (SOFTEX, 2012). Este trabalho propõe uma abordagem de ger{\^e}ncia de reutiliza{\c{c}}{\~a}o para apoiar o desenvolvimento de sistemas de gerenciamento integrado em uma empresa real, seguindo os preceitos do processo de Ger{\^e}ncia de Reutiliza{\c{c}}{\~a}o (GRU) do modelo de refer{\^e}ncia do MPS.BR (KALINOWSKI et al., 2010).},
  booktitle = {V Workshop do Departamento de Ci{\^e}ncia da Computa{\c{c}}{\~a}o da UFJF, Juiz de Fora, Brazil},
  pages     = {1},
  year      = {2013},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/SilvaK13.pdf},
  doi       = {},
}

Reutilização de software é o processo de criação de sistemas de software a partir de software existente ao invés de construí-los a partir do zero (KRUEGER, 1992). Sabe-se que a reutilização pode trazer benefícios para uma organização, como o aumento da qualidade e da produtividade e a redução dos custos e do tempo de desenvolvimento de sistemas. Neste contexto, a gerência de reutilização tem o propósito de gerenciar o ciclo de vida de ativos reutilizáveis, propondo um mecanismo de armazenamento para busca e recuperação dos mesmos (SOFTEX, 2012). Este trabalho propõe uma abordagem de gerência de reutilização para apoiar o desenvolvimento de sistemas de gerenciamento integrado em uma empresa real, seguindo os preceitos do processo de Gerência de Reutilização (GRU) do modelo de referência do MPS.BR (KALINOWSKI et al., 2010).

@article{KalinowskiCT12,
  author    = {Marcos Kalinowski and
               David N. Card and
               Guilherme Horta Travassos},
  title     = {Evidence-Based Guidelines to Defect Causal Analysis},
  abstract  = {Default causal analysis (DCA) or defect prevention is required by higher-maturity-level software development processes such as the Brazilian Software Process Improvement Reference Model and Capability Maturity Model Integration. The authors ask and answer questions about implementing it in lower-maturity organizations. In the related web extra entitled "Evidence-Based Guidelines on Defect Causal Analysis," authors Marcos Kalinowski, David N. Card, and Guilherme H. Travassos discuss the basics of research protocol. © 2012 IEEE.},  
  journal   = {{IEEE} Software},
  volume    = {29},
  number    = {4},
  pages     = {16-18},
  year      = {2012},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/KalinowskiCT12.pdf},
  doi       = {10.1109/MS.2012.72},
}

Default causal analysis (DCA) or defect prevention is required by higher-maturity-level software development processes such as the Brazilian Software Process Improvement Reference Model and Capability Maturity Model Integration. The authors ask and answer questions about implementing it in lower-maturity organizations. In the related web extra entitled "Evidence-Based Guidelines on Defect Causal Analysis," authors Marcos Kalinowski, David N. Card, and Guilherme H. Travassos discuss the basics of research protocol. © 2012 IEEE.

@inproceedings{SantosKRTWA12,
  author    = {Gleison Santos and
               Marcos Kalinowski and
               Ana Regina Rocha and
               Guilherme Horta Travassos and
               Kival Chaves Weber and
               Jos{\'{e}} Antonio Antonioni},
  title     = {{MPS.BR} Program and {MPS} Model: Main Results, Benefits and Beneficiaries
               of Software Process Improvement in Brazil},
  abstract  = {Over the past years Brazil has done big investments in software quality. In 2010 the investment was among the highest in the world. Among them, the long term MPS. BR program focuses on improving the software process quality of Brazilian companies. Therefore, this paper intends to describe its software process improvement reference model (MR-MPS) and the most important results accomplished to date on Government, Industry and University scenarios. The Brazilian Government had endorsed the MPS adoption as a mean to increase the software industry competitiveness both locally and globally. In the University scenario, the MPS model contributes to transfer software engineering knowledge to the Industry and allows the identification of new research challenges. The quantification of benefits in the Industry is based on the iMPS family of surveys which allow both performance results characterization and performance variation analysis considering organizations that adopted the MPS model. Quantitative data have shown the MPS model acceptance by the Industry. Moreover, the obtained results evidence improvements reduction of cost and schedule and increase of productivity and quality in the software projects. © 2012 IEEE.},			   
  booktitle = {8th International Conference on the Quality of Information and Communications
               Technology, {QUATIC} 2012, Lisbon, Portugal, 2-6 September 2012, Proceedings},
  pages     = {137-142},
  year      = {2012},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/SantosKRWA12.pdf},
  doi       = {10.1109/QUATIC.2012.42},
}

Over the past years Brazil has done big investments in software quality. In 2010 the investment was among the highest in the world. Among them, the long term MPS. BR program focuses on improving the software process quality of Brazilian companies. Therefore, this paper intends to describe its software process improvement reference model (MR-MPS) and the most important results accomplished to date on Government, Industry and University scenarios. The Brazilian Government had endorsed the MPS adoption as a mean to increase the software industry competitiveness both locally and globally. In the University scenario, the MPS model contributes to transfer software engineering knowledge to the Industry and allows the identification of new research challenges. The quantification of benefits in the Industry is based on the iMPS family of surveys which allow both performance results characterization and performance variation analysis considering organizations that adopted the MPS model. Quantitative data have shown the MPS model acceptance by the Industry. Moreover, the obtained results evidence improvements reduction of cost and schedule and increase of productivity and quality in the software projects. © 2012 IEEE.

@inproceedings{KalinowskiT12,
  author    = {Marcos Kalinowski and
               Guilherme Horta Travassos},
  title     = {Uma Abordagem Probabilística para An{\'a}lise Causal de Defeitos de Software},
  abstract  = {An{\'a}lise causal de defeitos tem se mostrado uma forma eficiente para melhoria de processos com base no produto. Nesta tese uma abordagem probabilística para an{\'a}lise causal, chamada DPPI (Defect Prevention-Based Process Improvement) foi elaborada com base em evid{\^e}ncias obtidas a partir de quatro rodadas de revis{\~a}o sistem{\'a}tica da literatura e feedback obtido de especialistas da {\'a}rea. DPPI representa uma abordagem inovadora que integra mecanismos de aprendizado de causa e efeito (redes Bayesianas) nos procedimentos de an{\'a}lise causal de defeitos. Adicionalmente, para facilitar o uso destes mecanismos em reuniões de an{\'a}lise causal, o tradicional diagrama de causa e efeito foi estendido para um diagrama de causa e efeito probabilístico. DPPI foi aplicada a um projeto real e avaliada atrav{\'e}s de tr{\^e}s rodadas de um estudo experimental. A aplica{\c{c}}{\~a}o ao projeto real indicou sua viabilidade e permitiu refinar requisitos para a constru{\c{c}}{\~a}o de apoio ferramental. As rodadas do estudo experimental forneceram indícios de que o uso dos diagramas de causa e efeito probabilísticos de DPPI aumenta a efic{\'a}cia e reduz o esfor{\c{c}}o na identifica{\c{c}}{\~a}o de causas de defeitos, quando comparado à identifica{\c{c}}{\~a}o de causas de defeitos sem o uso dos diagramas.},
  booktitle = {{SBQS} 2012 - Proceedings of the Brazilian Symposium on Software Quality, Fortaleza, Brazil},
  pages     = {414-428},
  year      = {2012},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/KalinowskiT12.pdf},
  doi       = {},
}

Análise causal de defeitos tem se mostrado uma forma eficiente para melhoria de processos com base no produto. Nesta tese uma abordagem probabilística para análise causal, chamada DPPI (Defect Prevention-Based Process Improvement) foi elaborada com base em evidências obtidas a partir de quatro rodadas de revisão sistemática da literatura e feedback obtido de especialistas da área. DPPI representa uma abordagem inovadora que integra mecanismos de aprendizado de causa e efeito (redes Bayesianas) nos procedimentos de análise causal de defeitos. Adicionalmente, para facilitar o uso destes mecanismos em reuniões de análise causal, o tradicional diagrama de causa e efeito foi estendido para um diagrama de causa e efeito probabilístico. DPPI foi aplicada a um projeto real e avaliada através de três rodadas de um estudo experimental. A aplicação ao projeto real indicou sua viabilidade e permitiu refinar requisitos para a construção de apoio ferramental. As rodadas do estudo experimental forneceram indícios de que o uso dos diagramas de causa e efeito probabilísticos de DPPI aumenta a eficácia e reduz o esforço na identificação de causas de defeitos, quando comparado à identificação de causas de defeitos sem o uso dos diagramas.

@book{TravassosK12b,
  author    = {Guilherme Horta Travassos and
               Marcos Kalinowski}, 
  title     = {iMPS 2012 – Evidence on Performance of Organizations that Adopted the MPS-SW Model since 2008 [ENGLISH VERSION]},
  publisher = {Softex},
  year      = 2012,
  note      = {ISBN: 978-85-99334-44-7, 36 p.},
  isbn      = {978-85-99334-44-7},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/TravassosK12b.pdf},
}

@book{TravassosK12c,
  author    = {Guilherme Horta Travassos and
               Marcos Kalinowski}, 
  title     = {iMPS 2012 – Evid{\^e}ncias Sobre o Desempenho das Empresas que Adotaram o Modelo MPS-SW desde 2008 [PORTUGUESE VERSION]},
  publisher = {Softex},
  year      = 2012,
  note      = {ISBN: 978-85-99334-44-7, 36 p.},
  isbn      = {978-85-99334-44-7},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/TravassosK12c.pdf},
}

@book{TravassosK12d,
  author    = {Guilherme Horta Travassos and
               Marcos Kalinowski}, 
  title     = {iMPS 2012 – Evidencias Sobre el Desempeño de las Empresas que Adoptaron el Modelo MPS-SW desde 2008 [SPANISH VERSION]},
  publisher = {Softex},
  year      = 2012,
  note      = {ISBN: 978-85-99334-44-7, 36 p.},
  isbn      = {978-85-99334-44-7},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/TravassosK12d.pdf},
}

@inproceedings{TravassosK12a,
  author    = {Guilherme Horta Travassos and
               Marcos Kalinowski},
  title     = {Resultados Iniciais da Rodada 5 do iMPS: Desempenho das Empresas que Adotaram o Modelo MPS},
  abstract  = {Este artigo apresenta uma descri{\c{c}}{\~a}o dos resultados da quinta rodada do iMPS (informa{\c{c}}ões para acompanhar e evidenciar varia{\c{c}}{\~a}o de desempenho nas empresas que adotaram o Modelo MPS). Neste quinto ano de coleta de dados, os resultados s{\~a}o apresentados sob duas perspectivas: a caracteriza{\c{c}}{\~a}o 2012 e uma an{\'a}lise global envolvendo o agrupamento dos dados das organiza{\c{c}}ões que participaram de pelo menos duas rodadas iMPS desde 2008. De forma geral, a satisfa{\c{c}}{\~a}o das empresas se mant{\'e}m alta, com mais de 95% das empresas informando estar totalmente ou parcialmente satisfeitas com o modelo MPS. Os dados de caracteriza{\c{c}}{\~a}o permitiram observar que as empresas de maior maturidade possuem mais projetos no país, tem maior presen{\c{c}}a no exterior, realizam entregas mais próximas aos prazos estimados, possuem maior capacidade produtiva (PF/M{\^e}s) e entregam produtos de maior qualidade (com menos defeitos). Os resultados globais iniciais indicam que as empresas em níveis de maturidade mais altos apresentam ganhos em rela{\c{c}}{\~a}o ao número de clientes, número de projetos no país, número de funcion{\'a}rios e melhoram a precis{\~a}o das estimativas. {\'e} possível observar varia{\c{c}}ões no tempo, esfor{\c{c}}o e tamanho m{\'e}dio dos projetos, com algum reflexo na produtividade mensal, apesar de que an{\'a}lises adicionais ainda necessitem ser realizadas de forma a fortalecer este resultado.},
  booktitle = {{WAMPS} 2012 - Workshop Anual do MPS.BR, Campinas, Brazil},
  pages     = {30-42},
  year      = {2012},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/TravassosK12a.pdf},
  doi       = {},
}

Este artigo apresenta uma descrição dos resultados da quinta rodada do iMPS (informações para acompanhar e evidenciar variação de desempenho nas empresas que adotaram o Modelo MPS). Neste quinto ano de coleta de dados, os resultados são apresentados sob duas perspectivas: a caracterização 2012 e uma análise global envolvendo o agrupamento dos dados das organizações que participaram de pelo menos duas rodadas iMPS desde 2008. De forma geral, a satisfação das empresas se mantém alta, com mais de 95% das empresas informando estar totalmente ou parcialmente satisfeitas com o modelo MPS. Os dados de caracterização permitiram observar que as empresas de maior maturidade possuem mais projetos no país, tem maior presença no exterior, realizam entregas mais próximas aos prazos estimados, possuem maior capacidade produtiva (PF/Mês) e entregam produtos de maior qualidade (com menos defeitos). Os resultados globais iniciais indicam que as empresas em níveis de maturidade mais altos apresentam ganhos em relação ao número de clientes, número de projetos no país, número de funcionários e melhoram a precisão das estimativas. é possível observar variações no tempo, esforço e tamanho médio dos projetos, com algum reflexo na produtividade mensal, apesar de que análises adicionais ainda necessitem ser realizadas de forma a fortalecer este resultado.

@phdthesis{Kalinowski11,
  author       = {Marcos Kalinowski}, 
  title        = {Uma Abordagem Probabilística para An{\'a}lise Causal de Defeitos de Software},
  school       = {Federal University of Rio de Janeiro {COPPE/UFRJ}},
  year         = 2011,
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/Kalinowski11.pdf},  
}

@inproceedings{KalinowskiMT11,
  author    = {Marcos Kalinowski and
               Emilia Mendes and
               Guilherme Horta Travassos},
  title     = {Automating and Evaluating Probabilistic Cause-Effect Diagrams to Improve
               Defect Causal Analysis},
  abstract  = {Defect causal analysis (DCA) has shown itself an efficient means to obtain product-focused software process improvement. A DCA approach, called DPPI, was assembled based on guidance acquired through systematic reviews and feedback from experts in the field. To our knowledge, DPPI represents an innovative approach integrating cause-effect learning mechanisms (Bayesian networks) into DCA meetings, by using probabilistic cause-effect diagrams. The experience of applying DPPI to a real Web-based software project showed its feasibility and provided insights into the requirements for tool support. Moreover, it was possible to observe that DPPI's Bayesian diagnostic inference predicted the main defect causes efficiently, motivating further investigation. This paper describes (i) the framework built to support the application of DPPI and automate the generation of the probabilistic cause-effect diagrams, and (ii) the results of an experimental study aiming at investigating the benefits of using DPPI's probabilistic cause-effect diagrams during DCA meetings. © 2011 Springer-Verlag.},			   
  booktitle = {Product-Focused Software Process Improvement - 12th International
               Conference, {PROFES} 2011, Torre Canne, Italy, June 20-22, 2011. Proceedings},
  pages     = {232-246},
  year      = {2011},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/KalinowskiMT11.pdf},
  doi       = {10.1007/978-3-642-21843-9_19},
}

Defect causal analysis (DCA) has shown itself an efficient means to obtain product-focused software process improvement. A DCA approach, called DPPI, was assembled based on guidance acquired through systematic reviews and feedback from experts in the field. To our knowledge, DPPI represents an innovative approach integrating cause-effect learning mechanisms (Bayesian networks) into DCA meetings, by using probabilistic cause-effect diagrams. The experience of applying DPPI to a real Web-based software project showed its feasibility and provided insights into the requirements for tool support. Moreover, it was possible to observe that DPPI's Bayesian diagnostic inference predicted the main defect causes efficiently, motivating further investigation. This paper describes (i) the framework built to support the application of DPPI and automate the generation of the probabilistic cause-effect diagrams, and (ii) the results of an experimental study aiming at investigating the benefits of using DPPI's probabilistic cause-effect diagrams during DCA meetings. © 2011 Springer-Verlag.

@inproceedings{KalinowskiSPRWA11,
  author    = {Marcos Kalinowski and
               Gleison Santos and
               Rafael Prikladnicki and
               Ana Regina Rocha and
               Kival Chaves Weber and
               Jos{\'{e}} Antonio Antonioni},
  title     = {From Software Engineering Research to Brazilian Software Quality Improvement},
  abstract  = {Software Engineering is a discipline that since its emergence is related to the solution of practical problems faced by the software industry. The academy has a fundamental role in finding solutions to the desires and needs of industry and thus to improve the quality of software produced. This paper presents the role of academia in the creation and transference of Software Engineering knowledge to the Brazilian industry, in particular by the results of the Brazilian Software Process Improvement Program (MPS.BR). The paper also highlights the Brazilian Symposium on Software Engineering (SBES) as a key forum for the establishment and strengthening of the Software Quality community in Brazil. © 2011 IEEE.},  
  booktitle = {25th Brazilian Symposium on Software Engineering, {SBES} 2011, Sao
               Paulo, Brazil, September 28-30, 2011},
  pages     = {120-125},
  year      = {2011},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/KalinowskiSPRWA11.pdf},
  doi       = {10.1109/SBES.2011.31},
}

Software Engineering is a discipline that since its emergence is related to the solution of practical problems faced by the software industry. The academy has a fundamental role in finding solutions to the desires and needs of industry and thus to improve the quality of software produced. This paper presents the role of academia in the creation and transference of Software Engineering knowledge to the Brazilian industry, in particular by the results of the Brazilian Software Process Improvement Program (MPS.BR). The paper also highlights the Brazilian Symposium on Software Engineering (SBES) as a key forum for the establishment and strengthening of the Software Quality community in Brazil. © 2011 IEEE.

@book{TravassosK11b,
  author    = {Guilherme Horta Travassos and
               Marcos Kalinowski}, 
  title     = {iMPS 2011 – Performance Results of Software Organizations that Adopted the MPS Model from 2008 to 2011 [ENGLISH VERSION]},
  publisher = {Softex},
  year      = 2011,
  note      = {ISBN: 978-85-99334-33-1, 36 p.},
  isbn      = {978-85-99334-33-1},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/TravassosK11b.pdf},
}

@book{TravassosK11c,
  author    = {Guilherme Horta Travassos and
               Marcos Kalinowski}, 
  title     = {iMPS 2011 – Resultados de Desempenho das Empresas que Adotaram o Modelo MPS de 2008 a 2011 [PORTUGUESE VERSION]},
  publisher = {Softex},
  year      = 2011,
  note      = {ISBN: 978-85-99334-33-1, 36 p.},
  isbn      = {978-85-99334-33-1},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/TravassosK11c.pdf},
}

@book{TravassosK11d,
  author    = {Guilherme Horta Travassos and
               Marcos Kalinowski}, 
  title     = {iMPS 2011 – Resultados de Desempeño de las Empresas que Adoptaron el Modelo MPS de 2008 a 2011 [SPANISH VERSION]},
  publisher = {Softex},
  year      = 2011,
  note      = {ISBN: 978-85-99334-33-1, 36 p.},
  isbn      = {978-85-99334-33-1},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/TravassosK11d.pdf},
}

@inproceedings{TravassosK11a,
  author    = {Guilherme Horta Travassos and
               Marcos Kalinowski},
  title     = {iMPS Rodada 4: Varia{\c{c}}{\~a}o de Desempenho nas Empresas que Adotaram o Modelo MPS de 2008 a 2011},
  abstract  = {Este artigo apresenta uma descri{\c{c}}{\~a}o dos resultados da quarta rodada do iMPS (informa{\c{c}}ões para acompanhar e evidenciar varia{\c{c}}{\~a}o de desempenho nas empresas que adotaram o Modelo MPS). Neste quarto ano de coleta de dados, os resultados s{\~a}o apresentados sob quatro perspectivas: caracteriza{\c{c}}{\~a}o 2011, an{\'a}lise de varia{\c{c}}{\~a}o 2010/2011, an{\'a}lise de varia{\c{c}}{\~a}o 2009/2010/2011 e an{\'a}lise de varia{\c{c}}{\~a}o 2008/2009/2010/2011.  De forma geral, a satisfa{\c{c}}{\~a}o das empresas se mant{\'e}m alta, com aproximadamente 97% das empresas informando estar totalmente ou parcialmente satisfeitas com o modelo MPS. Em rela{\c{c}}{\~a}o ao retorno de investimento da ado{\c{c}}{\~a}o do modelo, mais de 50% das empresas informaram ter recuperado nos últimos 12 meses mais do que o investimento feito na implementa{\c{c}}{\~a}o e avalia{\c{c}}{\~a}o do modelo. Os indicadores de varia{\c{c}}{\~a}o de desempenho para as empresas que adotaram o modelo MPS ao longo dos anos indicam que possivelmente empresas que se mant{\^e}m persistentes na utiliza{\c{c}}{\~a}o das pr{\'a}ticas da engenharia de software promovidas pelo modelo MPS tem sido capazes de lidar com projetos maiores, em maior número e com maior controle.},
  booktitle = {{WAMPS} 2011 - Workshop Anual do MPS.BR, Campinas, Brazil},
  pages     = {26-37},
  year      = {2011},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/TravassosK11a.pdf},
  doi       = {},
}

Este artigo apresenta uma descrição dos resultados da quarta rodada do iMPS (informações para acompanhar e evidenciar variação de desempenho nas empresas que adotaram o Modelo MPS). Neste quarto ano de coleta de dados, os resultados são apresentados sob quatro perspectivas: caracterização 2011, análise de variação 2010/2011, análise de variação 2009/2010/2011 e análise de variação 2008/2009/2010/2011. De forma geral, a satisfação das empresas se mantém alta, com aproximadamente 97% das empresas informando estar totalmente ou parcialmente satisfeitas com o modelo MPS. Em relação ao retorno de investimento da adoção do modelo, mais de 50% das empresas informaram ter recuperado nos últimos 12 meses mais do que o investimento feito na implementação e avaliação do modelo. Os indicadores de variação de desempenho para as empresas que adotaram o modelo MPS ao longo dos anos indicam que possivelmente empresas que se mantêm persistentes na utilização das práticas da engenharia de software promovidas pelo modelo MPS tem sido capazes de lidar com projetos maiores, em maior número e com maior controle.

@inproceedings{KalinowskiMCT10,
  author    = {Marcos Kalinowski and
               Emilia Mendes and
               David N. Card and
               Guilherme Horta Travassos},
  title     = {Applying {DPPI:} {A} Defect Causal Analysis Approach Using Bayesian
               Networks},
  abstract  = {Defect causal analysis (DCA) provides a means for product-focused software process improvement. A DCA approach, called DPPI (Defect Prevention-based Process Improvement), was assembled based on DCA guidance obtained from systematic reviews and on feedback gathered from experts in the field. According to the systematic reviews, and to our knowledge, DPPI represents the only approach that integrates cause-effect learning mechanisms (by using Bayesian networks) into DCA meetings. In this paper we extend the knowledge regarding the feasibility of using DPPI by the software industry, by describing the experience of applying it end-to-end to a real Web-based software project and providing additional industrial usage considerations. Building and using Bayesian networks in the context of DCA showed promising preliminary results and revealed interesting possibilities. © 2010 Springer-Verlag.},
  booktitle = {Product-Focused Software Process Improvement, 11th International Conference,
               {PROFES} 2010, Limerick, Ireland, June 21-23, 2010. Proceedings},
  pages     = {92-106},
  year      = {2010},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/KalinowskiMCT10.pdf},
  doi       = {10.1007/978-3-642-13792-1_9},
}

Defect causal analysis (DCA) provides a means for product-focused software process improvement. A DCA approach, called DPPI (Defect Prevention-based Process Improvement), was assembled based on DCA guidance obtained from systematic reviews and on feedback gathered from experts in the field. According to the systematic reviews, and to our knowledge, DPPI represents the only approach that integrates cause-effect learning mechanisms (by using Bayesian networks) into DCA meetings. In this paper we extend the knowledge regarding the feasibility of using DPPI by the software industry, by describing the experience of applying it end-to-end to a real Web-based software project and providing additional industrial usage considerations. Building and using Bayesian networks in the context of DCA showed promising preliminary results and revealed interesting possibilities. © 2010 Springer-Verlag.

@inproceedings{SantosKRTWA10,
  author    = {Gleison Santos and
               Marcos Kalinowski and
               Ana Regina Rocha and
               Guilherme Horta Travassos and
               Kival Chaves Weber and
               Jos{\'{e}} Antonio Antonioni},
  title     = {{MPS.BR:} {A} Tale of Software Process Improvement and Performance
               Results in the Brazilian Software Industry},
  abstract  = {In December 2003 a Brazilian nationwide program was created aiming at improving software processes. The MPS.BR Program is responsible for a great evolution in the software development scenario in Brazil by providing the means to support software process improvement initiatives based on the MPS Model. In 2008 a project named iMPS was launched to enable the analysis of the Performance Results of organizations that adopted the MPS Model. This paper describes some of the findings of the studies that have already been conducted. Preliminary evidences show improvement trends regarding cost, quality, schedule, and productivity, especially for those organizations that adopted the MPS Model for continuous improvement purposes. © 2010 IEEE.},			   
  booktitle = {Quality of Information and Communications Technology, 7th International
               Conference on the Quality of Information and Communications Technology,
               {QUATIC} 2010, Porto, Portugal, 29 September - 2 October, 2010, Proceedings},
  pages     = {412-417},
  year      = {2010},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/SantosKRTWA10.pdf},
  doi       = {10.1109/QUATIC.2010.75},
}

In December 2003 a Brazilian nationwide program was created aiming at improving software processes. The MPS.BR Program is responsible for a great evolution in the software development scenario in Brazil by providing the means to support software process improvement initiatives based on the MPS Model. In 2008 a project named iMPS was launched to enable the analysis of the Performance Results of organizations that adopted the MPS Model. This paper describes some of the findings of the studies that have already been conducted. Preliminary evidences show improvement trends regarding cost, quality, schedule, and productivity, especially for those organizations that adopted the MPS Model for continuous improvement purposes. © 2010 IEEE.

@inproceedings{KalinowskiSRMRWT10,
  author    = {Kalinowski, M. and Santos, G. and Reinehr, S. and Montoni, M. and Rocha, A.R. and Weber, K.C. and Travassos, G.H.},
  title     = {MPS.BR: Promoting the adoption of best software engineering practices by the Brazilian industry [MPS.BR: Promovendo a ado{\c{c}}{\~a}o de boas pr{\'a}ticas de engenharia de software pela indústria Brasileira]},
  abstract  = {The MPS model has been developed in the context of the MPS.BR Program in order to address the business needs of the Brazilian software industry. In this paper we present the current version of the MPS model, quantitative results of its adoption, and qualitative performance results obtained by software organizations that adopted the model, gathered through an experimental strategy based on surveys applied during two consecutive years. The quantitative results (174 organizations appraised until September 2009) point to an increasing adoption of the MPS model by Brazilian software organizations and to its capacity in promoting good software engineering practices. The qualitative results, on the other hand, show an increase of customer satisfaction and productivity, and capacity to deal with bigger projects for organizations that adopted the MPS model. © 2010 by CIbSE 2010.},  
  booktitle = {CIbSE 2010 - XIII Ibero-American Conference on Software Engineering},
  pages     = {265-278},
  year      = {2010},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/KalinowskiSRMRWT10.pdf},
}

The MPS model has been developed in the context of the MPS.BR Program in order to address the business needs of the Brazilian software industry. In this paper we present the current version of the MPS model, quantitative results of its adoption, and qualitative performance results obtained by software organizations that adopted the model, gathered through an experimental strategy based on surveys applied during two consecutive years. The quantitative results (174 organizations appraised until September 2009) point to an increasing adoption of the MPS model by Brazilian software organizations and to its capacity in promoting good software engineering practices. The qualitative results, on the other hand, show an increase of customer satisfaction and productivity, and capacity to deal with bigger projects for organizations that adopted the MPS model. © 2010 by CIbSE 2010.

@book{TravassosK10b,
  author    = {Guilherme Horta Travassos and
               Marcos Kalinowski}, 
  title     = {iMPS 2010 – Performance of Software Organizations that Adopted the MPS Model from 2008 to 2010 [ENGLISH VERSION]},
  publisher = {Softex},
  year      = 2010,
  note      = {ISBN: 978-85-99334-20-1, 32 p.},
  isbn      = {978-85-99334-20-1},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/TravassosK10b.pdf},
}

@book{TravassosK10c,
  author    = {Guilherme Horta Travassos and
               Marcos Kalinowski}, 
  title     = {iMPS 2010 – Desempenho das Empresas que Adotaram o Modelo MPS de 2008 a 2010 [PORTUGUESE VERSION]},
  publisher = {Softex},
  year      = 2010,
  note      = {ISBN: 978-85-99334-20-1, 32 p.},
  isbn      = {978-85-99334-20-1},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/TravassosK10c.pdf},
}

@book{TravassosK10d,
  author    = {Guilherme Horta Travassos and
               Marcos Kalinowski}, 
  title     = {iMPS 2010 – Desempeño de las Empresas que Adoptaron el Modelo MPS de 2008 a 2010 [SPANISH VERSION]},
  publisher = {Softex},
  year      = 2010,
  note      = {ISBN: 978-85-99334-20-1, 32 p.},
  isbn      = {978-85-99334-20-1},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/TravassosK10d.pdf},
}

@inproceedings{TravassosK10a,
  author    = {Guilherme Horta Travassos and
               Marcos Kalinowski},
  title     = {Resultados Iniciais iMPS 2010: Varia{\c{c}}{\~a}o de Desempenho nas Empresas que Adotaram o Modelo MPS},
  abstract  = {Este artigo apresenta uma síntese dos resultados iniciais da rodada de 2010 do projeto iMPS (informa{\c{c}}ões paraacompanhar e evidenciar varia{\c{c}}{\~a}o de desempenho nas empresas que adotaram o Modelo MPS). Neste terceiroano consecutivo de coleta de dados, os resultados s{\~a}o apresentados sob tr{\^e}s perspectivas: caracteriza{\c{c}}{\~a}o,an{\'a}lise de varia{\c{c}}{\~a}o 2009/2010 e an{\'a}lise de varia{\c{c}}{\~a}o 2008/2009/2010.  De forma geral, a satisfa{\c{c}}{\~a}o das empresascom o modelo {\'e} notória, com mais de 92% das empresas se dizendo parcialmente ou totalmente satisfeitas.Em rela{\c{c}}{\~a}o ao retorno de investimento da ado{\c{c}}{\~a}o do modelo, mais de 72% das empresas informaram terrecuperado mais do que o investimento feito na implementa{\c{c}}{\~a}o e avalia{\c{c}}{\~a}o do modelo. Al{\'e}m disso, assimcomo em 2009, principalmente para aquelas empresas que evoluíram ou internalizaram o MPS em seus processos,foi possível observar tend{\^e}ncias a redu{\c{c}}{\~a}o de custos, aumento da qualidade, redu{\c{c}}{\~a}o de prazos e aumento deprodutividade.},
  booktitle = {{WAMPS} 2010 - Workshop Anual do MPS.BR, Campinas, Brazil},
  pages     = {24-33},
  year      = {2010},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/TravassosK10a.pdf},
  doi       = {},
}

Este artigo apresenta uma síntese dos resultados iniciais da rodada de 2010 do projeto iMPS (informações paraacompanhar e evidenciar variação de desempenho nas empresas que adotaram o Modelo MPS). Neste terceiroano consecutivo de coleta de dados, os resultados são apresentados sob três perspectivas: caracterização,análise de variação 2009/2010 e análise de variação 2008/2009/2010. De forma geral, a satisfação das empresascom o modelo é notória, com mais de 92% das empresas se dizendo parcialmente ou totalmente satisfeitas.Em relação ao retorno de investimento da adoção do modelo, mais de 72% das empresas informaram terrecuperado mais do que o investimento feito na implementação e avaliação do modelo. Além disso, assimcomo em 2009, principalmente para aquelas empresas que evoluíram ou internalizaram o MPS em seus processos,foi possível observar tendências a redução de custos, aumento da qualidade, redução de prazos e aumento deprodutividade.

@article{RamiresSK10,
  author    = {Anderson Carlos Santos Ramires and
               Rodrigo Oliveira Sp{\'{\i}}nola and
               Marcos Kalinowski},
  title     = {Gest{\~a}o de TI: Auditoria de Sistemas},
  abstract  = {O objetivo principal deste artigo {\'e} identificar e comparar o conjunto de normas e padrões CobiT, ITIL e norma ISO 17799, emitidos por organismos internacionais, de natureza compulsória ou n{\~a}o, dos quais os auditores de se utilizam, para estudar e avaliar os sistemas, infraestrutura e controles internos automatizados no processo de auditoria dos sistemas que suportam as demonstra{\c{c}}ões cont{\'a}beis das entidades contratantes de tais servi{\c{c}}os.},  
  journal   = {Engenharia de Software Magazine},
  volume    = {28},
  number    = {},
  pages     = {26-37},
  year      = {2010},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/RamiresSK10.pdf},
  doi       = {},
}

O objetivo principal deste artigo é identificar e comparar o conjunto de normas e padrões CobiT, ITIL e norma ISO 17799, emitidos por organismos internacionais, de natureza compulsória ou não, dos quais os auditores de se utilizam, para estudar e avaliar os sistemas, infraestrutura e controles internos automatizados no processo de auditoria dos sistemas que suportam as demonstrações contábeis das entidades contratantes de tais serviços.

@article{BollerSCK10,
  author    = {Edmilson Rodrigues da Silva and
               Rodrigo Oliveira Sp{\'{\i}}nola and
               Marcos Kalinowski},
  title     = {Ger{\^e}ncia de Configura{\c{c}}{\~a}o: Defini{\c{c}}ões Iniciais, Ferramentas e Processo},
  abstract  = {Neste artigo ser{\~a}o mostrados inicialmente aspectos relevantes da ger{\^e}ncia de configura{\c{c}}{\~a}o. Na segunda parte do artigo, ser{\'a} apresentada a aplica{\c{c}}{\~a}o da ger{\^e}ncia de configura{\c{c}}{\~a}o dentro dos principais modelos de maturidade e as vantagens de se utilizar tais ferramentas, atrav{\'e}s de um estudo de caso de implementa{\c{c}}{\~a}o em uma empresa hipot{\'e}tica.},  
  journal   = {Engenharia de Software Magazine},
  volume    = {24},
  number    = {},
  pages     = {14-30},
  year      = {2010},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/BollerSCK10.pdf},
  doi       = {},
}

Neste artigo serão mostrados inicialmente aspectos relevantes da gerência de configuração. Na segunda parte do artigo, será apresentada a aplicação da gerência de configuração dentro dos principais modelos de maturidade e as vantagens de se utilizar tais ferramentas, através de um estudo de caso de implementação em uma empresa hipotética.

@book{TravassosK09b,
  author    = {Guilherme Horta Travassos and
               Marcos Kalinowski}, 
  title     = {iMPS 2009 – Characterization and Performance Variation of Software Organizations that Adopted the MPS Model [ENGLISH VERSION]},
  publisher = {Softex},
  year      = 2009,
  note      = {ISBN: 978-85-99334-18-8, 27 p.},
  isbn      = {978-85-99334-18-8},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/TravassosK09b.pdf},
}

@book{TravassosK09c,
  author    = {Guilherme Horta Travassos and
               Marcos Kalinowski}, 
  title     = {iMPS 2009 – Caracteriza{\c{c}}{\~a}o e Varia{\c{c}}{\~a}o de Desempenho de Organiza{\c{c}}ões que Adotaram o Modelo MPS [PORTUGUESE VERSION]},
  publisher = {Softex},
  year      = 2009,
  note      = {ISBN: 978-85-99334-18-8, 27 p.},
  isbn      = {978-85-99334-18-8},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/TravassosK09c.pdf},
}

@book{TravassosK09d,
  author    = {Guilherme Horta Travassos and
               Marcos Kalinowski}, 
  title     = {iMPS 2009 – Caracterización y Variación de Desempeño de Organizaciones que Adoptaron el Modelo MPS [SPANISH VERSION]},
  publisher = {Softex},
  year      = 2009,
  note      = {ISBN: 978-85-99334-18-8, 27 p.},
  isbn      = {978-85-99334-18-8},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/TravassosK09d.pdf},
}

@inproceedings{TravassosK09a,
  author    = {Guilherme Horta Travassos and
               Marcos Kalinowski},
  title     = {Varia{\c{c}}{\~a}o de Desempenho nas Empresas que Adotaram o Modelo MPS: Resultados Iniciais iMPS 2009},
  abstract  = {Este artigo apresenta uma síntese dos resultados iniciais da rodada de 2009 do projeto iMPS (in-forma{\c{c}}ões para acompanhar e evidenciar varia{\c{c}}{\~a}o de desempenho nas empresas que adotaram o Modelo MPS), apresentados sob duas perspectivas: caracteriza{\c{c}}{\~a}o e an{\'a}lise de varia{\c{c}}{\~a}o 2008/2009. De forma geral, a satisfa{\c{c}}{\~a}o das empresas com o modelo {\'e} notória, com mais de 98% das empresas se dizendo parcialmente ou totalmente satisfeitas. Al{\'e}m disso, as empresas relataram que o retorno do investimento foi obtido e, principal-mente, para aquelas empresas que evoluíram ou internalizaram o MPS em seus processos foi possível observar tend{\^e}ncia a melhoria de custo, qualidade, prazo e produtividade, princípios b{\'a}sicos quando se desenvolve software seguindo os preceitos de engenharia.},
  booktitle = {{WAMPS} 2009 - Workshop Anual do MPS.BR, Campinas, Brazil},
  pages     = {20-29},
  year      = {2009},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/TravassosK09a.pdf},
  doi       = {},
}

Este artigo apresenta uma síntese dos resultados iniciais da rodada de 2009 do projeto iMPS (in-formações para acompanhar e evidenciar variação de desempenho nas empresas que adotaram o Modelo MPS), apresentados sob duas perspectivas: caracterização e análise de variação 2008/2009. De forma geral, a satisfação das empresas com o modelo é notória, com mais de 98% das empresas se dizendo parcialmente ou totalmente satisfeitas. Além disso, as empresas relataram que o retorno do investimento foi obtido e, principal-mente, para aquelas empresas que evoluíram ou internalizaram o MPS em seus processos foi possível observar tendência a melhoria de custo, qualidade, prazo e produtividade, princípios básicos quando se desenvolve software seguindo os preceitos de engenharia.

@article{CostaKS09,
  author    = {Marcelo Nascimento Costa and
               Marcos Kalinowski and
			   Rodrigo Oliveira Sp{\'{\i}}nola},
  title     = {Solu{\c{c}}ões Concretas para Problemas Pr{\'a}ticos da Engenharia de Requisitos},
  abstract  = {O artigo apresenta uma coletânea de problemas pr{\'a}ticos e discute solu{\c{c}}ões que tipicamente tem sido empregadas para tratar esses problemas},  
  journal   = {Engenharia de Software Magazine},
  volume    = {13},
  number    = {},
  pages     = {8-15},
  year      = {2009},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/CostaKS09.pdf},
  doi       = {},
}

O artigo apresenta uma coletânea de problemas práticos e discute soluções que tipicamente tem sido empregadas para tratar esses problemas

@inproceedings{KalinowskiTC08a,
  author    = {Marcos Kalinowski and
               Guilherme Horta Travassos and
               David N. Card},
  title     = {Towards a Defect Prevention Based Process Improvement Approach},
  abstract  = {Defect causal analysis (DCA) is a means of product focused software process improvement. A systematic literature review to identify the DCA state of the art has been undertaken. The systematic review gathered unbiased knowledge and evidence and identified opportunities for further investigation. Moreover, some guidance on how to efficiently implement DCA in software organizations could be elaborated. This paper describes the initial concept of the DBPI (Defect Based Process Improvement) approach. It represents a DCA based approach for process improvement, designed considering the results of the systematic review and the obtained guidance. Its main contributions are tailoring support for DCA based process improvement and addressing an identified opportunity for further investigation by integrating organizational learning mechanisms regarding causeeffect relations into the conduct of DCA. © 2008 IEEE.},
  booktitle = {34th Euromicro Conference on Software Engineering and Advanced Applications,
               {SEAA} 2008, September 3-5, 2008, Parma, Italy},
  pages     = {199-206},
  year      = {2008},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/KalinowskiTC08a.pdf},
  doi       = {10.1109/SEAA.2008.47},
}

Defect causal analysis (DCA) is a means of product focused software process improvement. A systematic literature review to identify the DCA state of the art has been undertaken. The systematic review gathered unbiased knowledge and evidence and identified opportunities for further investigation. Moreover, some guidance on how to efficiently implement DCA in software organizations could be elaborated. This paper describes the initial concept of the DBPI (Defect Based Process Improvement) approach. It represents a DCA based approach for process improvement, designed considering the results of the systematic review and the obtained guidance. Its main contributions are tailoring support for DCA based process improvement and addressing an identified opportunity for further investigation by integrating organizational learning mechanisms regarding causeeffect relations into the conduct of DCA. © 2008 IEEE.

@inproceedings{KalinowskiWT08,
  author    = {Marcos Kalinowski and
               Kival Chaves Weber and
               Guilherme Horta Travassos},
  title     = {{IMPS:} an experimentation based investigation of a nationwide software
               development reference model},
  abstract  = {Experimentation in software engineering represents a means to understand the impact of adopting software technologies. In this paper we describe an on-going experimentation work aiming at to address a real industry need in Brazil: the understanding of how performance variables (such as cost, productivity, and quality) are affected by the adoption of a nationwide software development reference model. The main strategy consists in integrating the application of carefully planned surveys into the model implementation and assessment procedures. Besides the expected performance results, we hope it can be useful to reinforce to the Brazilian software industry the importance of experimentation to support software technologies evaluation. Copyright 2008 ACM.},
  booktitle = {Proceedings of the Second International Symposium on Empirical Software
               Engineering and Measurement, {ESEM} 2008, October 9-10, 2008, Kaiserslautern,
               Germany},
  pages     = {336-338},
  year      = {2008},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/KalinowskiWT08.pdf},
  doi       = {10.1145/1414004.1414073},
}

Experimentation in software engineering represents a means to understand the impact of adopting software technologies. In this paper we describe an on-going experimentation work aiming at to address a real industry need in Brazil: the understanding of how performance variables (such as cost, productivity, and quality) are affected by the adoption of a nationwide software development reference model. The main strategy consists in integrating the application of carefully planned surveys into the model implementation and assessment procedures. Besides the expected performance results, we hope it can be useful to reinforce to the Brazilian software industry the importance of experimentation to support software technologies evaluation. Copyright 2008 ACM.

@inproceedings{KalinowskiTC08b,
  author    = {Marcos Kalinowski and
               Guilherme Horta Travassos and
			   David N. Card},
  title     = {Guidance for Efficiently Implementing Defect Causal Analysis},
  abstract  = {Defect causal analysis has shown itself to be a cheap and high return means of product-focused software process improvement. However, despite its advantages and wide industry adoption little academic research is being done in this area. Thus, professionals face several questions when implementing it in software organizations. Aiming to provide unbiased and evidence-based answers to those questions, a systematic review has been conducted. Based on the results of the systematic review, better guidance for implementing defect causal analysis efficiently in software organizations can be elaborated.},
  booktitle = {{SBQS} 2008 - Proceedings of the Brazilian Symposium on Software Quality, Florianopolis, Brazil},
  pages     = {1-15},
  year      = {2008},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/KalinowskiTC08b.pdf},
  doi       = {},
}

Defect causal analysis has shown itself to be a cheap and high return means of product-focused software process improvement. However, despite its advantages and wide industry adoption little academic research is being done in this area. Thus, professionals face several questions when implementing it in software organizations. Aiming to provide unbiased and evidence-based answers to those questions, a systematic review has been conducted. Based on the results of the systematic review, better guidance for implementing defect causal analysis efficiently in software organizations can be elaborated.

@inproceedings{KalinowskiT08,
  author    = {Marcos Kalinowski and
               Guilherme Horta Travassos},
  title     = {Towards a Defect Causal Analysis Approach for Software Process Improvement and Organizational Learning},
  abstract  = {Defect causal analysis is expected to improve quality and productivity by addressing process improvement in order to prevent product defects. A systematic review to identify defect causal analysis state of the art has been undertaken. Besides guidance and suggestions on how to implement it efficiently in software organizations it has also identified opportunities for further investigation. For instance, the systematic review was not able to find any approach embedding learning mechanisms concerning the cause-effect relationships reflecting the organizational context reality. Such mechanisms can enable organizational learning and, for instance, can make the identification of causes and action proposals easier. Therefore, we propose a doctoral research concerned with providing an approach for defect causal analysis based on unbiased knowledge and evidence regarding defect causal analysis best practices. It aims at filling the identified gap integrating organizational learning into the conduction of defect causal analysis.},
  booktitle = {{SBQS} 2008 - Doctoral Symposium of the Brazilian Symposium on Software Quality, Florianopolis, Brazil},
  pages     = {1-8},
  year      = {2008},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/KalinowskiT08.pdf},
  doi       = {},
}

Defect causal analysis is expected to improve quality and productivity by addressing process improvement in order to prevent product defects. A systematic review to identify defect causal analysis state of the art has been undertaken. Besides guidance and suggestions on how to implement it efficiently in software organizations it has also identified opportunities for further investigation. For instance, the systematic review was not able to find any approach embedding learning mechanisms concerning the cause-effect relationships reflecting the organizational context reality. Such mechanisms can enable organizational learning and, for instance, can make the identification of causes and action proposals easier. Therefore, we propose a doctoral research concerned with providing an approach for defect causal analysis based on unbiased knowledge and evidence regarding defect causal analysis best practices. It aims at filling the identified gap integrating organizational learning into the conduction of defect causal analysis.

@book{TravassosK08,
  author    = {Guilherme Horta Travassos and
               Marcos Kalinowski}, 
  title     = {iMPS - Resultados de Desempenho de organiza{\c{c}}ões que adotaram o modelo MPS},
  publisher = {Softex},
  year      = {2008},
  note      = {ISBN: 978-85-99334-11-9, 28 p.},
  isbn      = {978-85-99334-11-9},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/TravassosK08.pdf},
}

@article{RochaEtAl08,
	author = {Ana Regina Rocha and Ahilton Barreto and Ana Cândida Natali and Andrea Barreto and Gleison Santos and Leonardo Murta and Marcos Kalinowski and Mariano Montoni and Paula Mian and Reinaldo Cabral and S{\'a}vio Figueiredo and Tayana Conte},
	title = {IA COPPE/UFRJ: Li{\c{c}}ões Aprendidas em 2008},
	journal = {ProQuality (UFLA)},
	volume = {4},
	number = {2},
	year = {2008},
	keywords = {},
	abstract = {A Institui{\c{c}}{\~a}o Avaliadora COPPE/UFRJ foi credenciada pela SOFTEX em 2006 e nestes dois anos realizou avalia{\c{c}}ões do nível G ao nível A, acumulando experi{\^e}ncias significativas. Em 2008 a equipe da COPPE foi avaliada MPS Nível E, e esta nova experi{\^e}ncia foi uma ocasi{\~a}o privilegiada para coletar li{\c{c}}ões aprendidas e faz{\^e}-las refletir na pr{\'a}tica da Institui{\c{c}}{\~a}o Avaliadora. Este artigo relata o conjunto de li{\c{c}}ões aprendidas pela equipe da COPPE a partir desta experi{\^e}ncia de ter sido avaliada como organiza{\c{c}}{\~a}o desenvolvedora de software.},
	issn = {1807-5061},	
	pages = {13-18},
	urlAuthor_version = {http://www.inf.puc-rio.br/~kalinowski/publications/RochaEtAl08.pdf},
}

A Instituição Avaliadora COPPE/UFRJ foi credenciada pela SOFTEX em 2006 e nestes dois anos realizou avaliações do nível G ao nível A, acumulando experiências significativas. Em 2008 a equipe da COPPE foi avaliada MPS Nível E, e esta nova experiência foi uma ocasião privilegiada para coletar lições aprendidas e fazê-las refletir na prática da Instituição Avaliadora. Este artigo relata o conjunto de lições aprendidas pela equipe da COPPE a partir desta experiência de ter sido avaliada como organização desenvolvedora de software.

@article{MunhaoCKS08,
  author    = {Augusto C{\'e}sar Brauns Munhao and
               Adriano Claudio Costa Campos and
               Marcos Kalinowski and
			   Rodrigo Oliveira Sp{\'{\i}}nola},
  title     = {Apoiando a Implementa{\c{c}}{\~a}o do Modelo MPS Nível G},
  abstract  = {O objetivo deste artigo {\'e} apresentar os resultados de uma pesquisa de campo (survey) para apontar as pr{\'a}ticas mais adotadas pelos implementadores do modelo MPS ao evidenciar as exig{\^e}ncias do nível G deste modelo.},  
  journal   = {Engenharia de Software Magazine},
  volume    = {7},
  number    = {},
  pages     = {50-67},
  year      = {2008},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/MunhaoCKS08.pdf},
  doi       = {},
}

O objetivo deste artigo é apresentar os resultados de uma pesquisa de campo (survey) para apontar as práticas mais adotadas pelos implementadores do modelo MPS ao evidenciar as exigências do nível G deste modelo.

@article{KalinowskiC08,
  author    = {Marcos Kalinowski and
               Marcelo Nascimento Costa},
  title     = {Melhorando Processos de Software atrav{\'e}s de An{\'a}lise Causal de Defeitos},
  abstract  = {Este artigo fornece uma vis{\~a}o geral de como a an{\'a}lise causal de defeitos pode ser aplicada para obter melhoria de processo baseada no produto. A base teórica a respeito de an{\'a}lise causal de defeitos {\'e} descrita, bem como maneiras efi cientes de implementar a mesma em organiza{\c{c}}ões de software. Adicionalmente, um exemplo real de sua aplica{\c{c}}{\~a}o na indústria {\'e} apresentado. Considerando que apenas dados relacionados a defeitos precisam ser coletados, a an{\'a}lise causal de defeitos {\'e} indicada mesmo para empresas com baixo nível de maturidade, permitindo a estas reduzir signifi cativamente suas taxas de defeitos e conseqüentemente o custo e tempo de desenvolvimento.},  
  journal   = {Engenharia de Software Magazine},
  volume    = {3},
  number    = {},
  pages     = {32-37},
  year      = {2008},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/KalinowskiC08.pdf},
  doi       = {},
}

Este artigo fornece uma visão geral de como a análise causal de defeitos pode ser aplicada para obter melhoria de processo baseada no produto. A base teórica a respeito de análise causal de defeitos é descrita, bem como maneiras efi cientes de implementar a mesma em organizações de software. Adicionalmente, um exemplo real de sua aplicação na indústria é apresentado. Considerando que apenas dados relacionados a defeitos precisam ser coletados, a análise causal de defeitos é indicada mesmo para empresas com baixo nível de maturidade, permitindo a estas reduzir signifi cativamente suas taxas de defeitos e conseqüentemente o custo e tempo de desenvolvimento.

@book{MPSBR_Guia_Geral,
  author    = {Ana Regina Rocha and Ana Liddy Magalh{\~a}es and Kathia Marcal de Oliveira and Mariano Montoni and Ahilton Barreto and Alfredo Nozomu Tsukumo and Claudia Werner and Gleison Santos and Leonardo Murta and Marco Lopes and Marcos Kalinowski}, 
  title     = {MPS.BR - Melhoria de Processo do Software Brasileiro - Guia Geral - Vers{\~a}o 1.2},
  publisher = {Softex},
  year      = {2007},
  note      = {ISBN: 978-85-99334-08-9, 52 p.},
  isbn      = {978-85-99334-08-9},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/MPSBR_Guia_Geral.pdf},
}

@book{MPSBR_Guia_Implementacao_Nivel_A,
  author    = {Ana Regina Rocha and Mariano Montoni and Gleison Santos and Marcos Kalinowski}, 
  title     = {MPS.BR - Melhoria de Processo do Software Brasileiro - Guia de Implementa{\c{c}}{\~a}o – Parte 7: Nível A - Vers{\~a}o 1.0},
  publisher = {Softex},
  year      = {2007},
  note      = {ISBN: 978-85-99334-55-3, 25 p.},
  isbn      = {978-85-99334-55-3},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/MPSBR_Guia_Implementacao_Nivel_A.pdf},
}

@inproceedings{KalinowskiTO07,
  author    = {Marcos Kalinowski and Hugo Vidal Teixeira and Paul Johan Heinrich van Oppen},
  title     = {ABAT: An Approach for Building Maintainable Automated Functional Software Tests},
  abstract  = {Statistical process control has been used in order to control the performance of software processes. However, this technique is limited to control the value of a metric in a current project, compared to the same metric considering the process performance. In this paper we present a quantitative project management methodology combining statistical process control and statistical performance models to allow predicting behaviors for different scenarios to support managerial decisions. The methodology was developed during the application of those techniques in a software organization to control and relate the Quality Assurance and Verification processes.},
  booktitle = {XXVI International Conference of the Chilean Computer Science Society (SCCC 2007), 8-9 November 2007, Iquique, Chile},
  pages     = {83-91},
  year      = {2007},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/KalinowskiTO07.pdf},
  doi       = {10.1109/SCCC.2007.3},
}

Statistical process control has been used in order to control the performance of software processes. However, this technique is limited to control the value of a metric in a current project, compared to the same metric considering the process performance. In this paper we present a quantitative project management methodology combining statistical process control and statistical performance models to allow predicting behaviors for different scenarios to support managerial decisions. The methodology was developed during the application of those techniques in a software organization to control and relate the Quality Assurance and Verification processes.

@inproceedings{KalinowskiT07,
  author    = {Marcos Kalinowski and Guilherme Horta Travassos},
  title     = {ISPIS: From Conception towards Industry Readiness},
  booktitle = {XXVI International Conference of the Chilean Computer Science Society (SCCC 2007), 8-9 November 2007, Iquique, Chile},
  pages     = {132-141},
  year      = {2007},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/KalinowskiT07.pdf},
  doi       = {10.1109/SCCC.2007.12},
}

@inproceedings{MontoniKLAFR07,
  author    = {Mariano Montoni and
               Marcos Kalinowski and
               Peter Lupo and
               Jose Fortuna Abrantes and
               Analia Ferreira and
			   Ana Regina Rocha},
  title     = {Uma Metodologia para Desenvolvimento de Modelos de Desempenho de Processos para Ger{\^e}ncia Quantitativa de Projetos de Software},
  abstract  = {Statistical process control has been used in order to control the performance of software processes. However, this technique is limited to control the value of a metric in a current project, compared to the same metric considering the process performance. In this paper we present a quantitative project management methodology combining statistical process control and statistical performance models to allow predicting behaviors for different scenarios to support managerial decisions. The methodology was developed during the application of those techniques in a software organization to control and relate the Quality Assurance and Verification processes.},
  booktitle = {{SBQS} 2007 - Proceedings of the Brazilian Symposium on Software Quality, Porto de Galinhas, Brazil},
  pages     = {325-339},
  year      = {2007},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/MontoniKLAFR07.pdf},
  doi       = {},
}

Statistical process control has been used in order to control the performance of software processes. However, this technique is limited to control the value of a metric in a current project, compared to the same metric considering the process performance. In this paper we present a quantitative project management methodology combining statistical process control and statistical performance models to allow predicting behaviors for different scenarios to support managerial decisions. The methodology was developed during the application of those techniques in a software organization to control and relate the Quality Assurance and Verification processes.

@inproceedings{KalinowskiSNBT07,
  author    = {Marcos Kalinowski and
               Rodrigo Oliveira Sp{\'{\i}}nola and
               Arilo Claudio Dias-Neto and
               Andrew Bott and
			   Guilherme Horta Travassos},
  title     = {Inspe{\c{c}}ões de Requisitos de Software em Desenvolvimento Incremental: Uma Experi{\^e}ncia Pr{\'a}tica},
  abstract  = {Software inspection is an efficient technique to reduce rework and improve the quality of software artifacts. However, in industry inspections are often conducted in a non-systematic way and their full potential is seldom achieved. In this paper we report the experience of institutionalizing the software inspection process in order to accomplish requirements inspections in a real project with an incremental life cycle. Among the obtained results we highlight the identification of opportunities for improving the software process, based on the analysis of the inspection results previous increments.},
  booktitle = {{SBQS} 2007 - Proceedings of the Brazilian Symposium on Software Quality, Porto de Galinhas, Brazil},
  pages     = {389-396},
  year      = {2007},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/KalinowskiSNBT07.pdf},
  doi       = {},
}

Software inspection is an efficient technique to reduce rework and improve the quality of software artifacts. However, in industry inspections are often conducted in a non-systematic way and their full potential is seldom achieved. In this paper we report the experience of institutionalizing the software inspection process in order to accomplish requirements inspections in a real project with an incremental life cycle. Among the obtained results we highlight the identification of opportunities for improving the software process, based on the analysis of the inspection results previous increments.

@inproceedings{XexeoKR07,
  author    = {Jose Antonio Moreira Xexeo and
               Marcos Kalinowski and
			   Clevi Rapkiewicz},
  title     = {Construindo Comunidades Virtuais de Aprendizagem: Uma Experi{\^e}ncia Pr{\'a}tica em Curso de Ci{\^e}ncia da Computa{\c{c}}{\~a}o},
  abstract  = {Establishing virtual learning communities in distance learning projects implies facing difficulties that are more closely related to the impact caused by an innovative project in the context of social systems that involve interacting people than to the technologies used and their technical details. However, in literature the technical contributions prevail and social interactive systems are still poorly understood. In this paper we analyze a distance learning experience, focusing on the social interactive system, in the specific context of a discipline with 57 freshman night shift computer science students of Metodista do Rio.},
  booktitle = {{WEI} 2007 - XV Workshop sobre Educa{\c{c}}{\~a}o em Computa{\c{c}}{\~a}o, Rio de Janeiro, Brazil},
  pages     = {1-10},
  year      = {2007},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/XexeoKR07.pdf},
  doi       = {},
}

Establishing virtual learning communities in distance learning projects implies facing difficulties that are more closely related to the impact caused by an innovative project in the context of social systems that involve interacting people than to the technologies used and their technical details. However, in literature the technical contributions prevail and social interactive systems are still poorly understood. In this paper we analyze a distance learning experience, focusing on the social interactive system, in the specific context of a discipline with 57 freshman night shift computer science students of Metodista do Rio.

@inproceedings{KalinowskiXS07,
  author    = {Marcos Kalinowski and
               Jos{\'e} Antonio Moreira Xexeo and
			   Gabriela Bastos Soares},
  title     = {Constru{\c{c}}{\~a}o de um Projeto Pedagógico de Ci{\^e}ncia da Computa{\c{c}}{\~a}o com {\^e}nfase em Engenharia de Software},
  abstract  = {This paper describes the development of a computer science curricula with emphasis on software engineering for the night shift course of Metodista do Rio in order to prepare the former students for both, the increasing demand in the software development industry and graduate courses. Moreover, the instantiation of such curricula is described and results of preliminary evaluations to obtain the awareness of students to the usefulness of two disciplines of the new curricula to develop their skills in the field of software development are presented.},
  booktitle = {{WEIMIG} 2007 - Workshop de Educa{\c{c}}{\~a}o em Computa{\c{c}}{\~a}o e Inform{\'a}tica do Estado de Minas Gerais, Lavras, Brazil},
  pages     = {1-10},
  year      = {2007},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/KalinowskiXS07.pdf},
  doi       = {},
}

This paper describes the development of a computer science curricula with emphasis on software engineering for the night shift course of Metodista do Rio in order to prepare the former students for both, the increasing demand in the software development industry and graduate courses. Moreover, the instantiation of such curricula is described and results of preliminary evaluations to obtain the awareness of students to the usefulness of two disciplines of the new curricula to develop their skills in the field of software development are presented.

@article{KalinowskiS07,
  author    = {Marcos Kalinowski and
               Rodrigo Oliveira Sp{\'{\i}}nola},
  title     = {Introdu{\c{c}}{\~a}o à Inspe{\c{c}}{\~a}o de Software},
  abstract  = {O artigo apresenta uma introdu{\c{c}}{\~a}o sobre como aumentr a qualidade atrav{\'e}s de verifica{\c{c}}ões intermedi{\'a}rias de artefatos de software. As orienta{\c{c}}ões providas s{\~a}o baseadas em evid{\^e}ncias da literatura na {\'a}rea de inspe{\c{c}}ões de software.},  
  journal   = {Engenharia de Software Magazine},
  volume    = {1},
  number    = {},
  pages     = {68-74},
  year      = {2007},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/KalinowskiS07.pdf},
  doi       = {},
}

O artigo apresenta uma introdução sobre como aumentr a qualidade através de verificações intermediárias de artefatos de software. As orientações providas são baseadas em evidências da literatura na área de inspeções de software.

@inproceedings{KalinowskiBT06,
  author    = {Marcos Kalinowski and
			   Marcos Roberto da Silva Borges and
			   Guilherme Horta Travassos},
  title     = {PatternFlow: Supporting Standardized Description and Enactment of Business Processes},
  abstract  = {The establishment of patterns to systematically address workflow requirements is important, since it should be possible for a modeled process to be described and enacted by workflow management systems. Despite the sprouting of those workflow patterns, many of them are not supported in a great amount of workflow modeling languages and workflow management systems. Moreover, many of those patterns recur quite frequently during the analysis phase of workflow projects, which highlights the importance of supporting such patterns. In this paper we present PatternFlow, a framework for describing and enacting business processes whose requirements set was derived mainly from knowledge acquired from the analysis of documented workflow patterns. PatternFlow has been used, among others, to implement ISPIS, a framework for supporting the software inspection process. Evaluations of ISPIS in academy and industry allowed us to gain additional insight regarding the feasibility of using PatternFlow for enacting processes used in real industrial environments.},
  booktitle = {{SBSC} 2006 - Proceedings of the Brazilian Symposium on Collaborative Systems, Natal, Brazil},
  pages     = {87-97},
  year      = {2006},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/KalinowskiBT06.pdf},  
  doi       = {},
}

The establishment of patterns to systematically address workflow requirements is important, since it should be possible for a modeled process to be described and enacted by workflow management systems. Despite the sprouting of those workflow patterns, many of them are not supported in a great amount of workflow modeling languages and workflow management systems. Moreover, many of those patterns recur quite frequently during the analysis phase of workflow projects, which highlights the importance of supporting such patterns. In this paper we present PatternFlow, a framework for describing and enacting business processes whose requirements set was derived mainly from knowledge acquired from the analysis of documented workflow patterns. PatternFlow has been used, among others, to implement ISPIS, a framework for supporting the software inspection process. Evaluations of ISPIS in academy and industry allowed us to gain additional insight regarding the feasibility of using PatternFlow for enacting processes used in real industrial environments.

@inproceedings{KalinowskiT05,
  author    = {Marcos Kalinowski and
			   Guilherme Horta Travassos},
  title     = {Software Technologies: The Use of Experimentation to Introduce ISPIS - A Software Inspection Framework - Into the Industry},
  abstract  = {ISPIS, a framework for supporting software inspection processes, resulted from academic research and was carefully evaluated into this context by experimental studies. In this paper we provide an overview on ISPIS and describe how it has been introduced into a software engineering organization. A methodology based on experimentation was considered in this case resulting in two additional case studies: ‘use in real life cycle’ and ‘use in industry’, respectively. The feedback from these two case studies resulted in adjustments demand by the practitioner’s on ISPIS. The main purpose of those adjustments was to increase the efficiency of software inspections by adding awareness support and carefully granting rights to participants in order to move towards the inspection conclusion.},
  booktitle = {{ESELAW} 2005 - Experimental Software Engineering Latin American Workshop, Uberlandia, Brazil},
  pages     = {1-6},
  year      = {2005},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/KalinowskiT05.pdf},    
  doi       = {},
}

ISPIS, a framework for supporting software inspection processes, resulted from academic research and was carefully evaluated into this context by experimental studies. In this paper we provide an overview on ISPIS and describe how it has been introduced into a software engineering organization. A methodology based on experimentation was considered in this case resulting in two additional case studies: ‘use in real life cycle’ and ‘use in industry’, respectively. The feedback from these two case studies resulted in adjustments demand by the practitioner’s on ISPIS. The main purpose of those adjustments was to increase the efficiency of software inspections by adding awareness support and carefully granting rights to participants in order to move towards the inspection conclusion.

@mastersthesis{Kalinowski04,
  author    = {Marcos Kalinowski}, 
  title     = {Infraestrutura Computacional de Apoio ao Processo de Inspe{\c{c}}{\~a}o de Software},
  school    = {Federal University of Rio de Janeiro {COPPE/UFRJ}},
  year      = {2004},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/Kalinowski04.pdf},
}

@inproceedings{KalinowskiT04,
  author    = {Marcos Kalinowski and
               Guilherme Horta Travassos},
  title     = {A Computational Framework for Supporting Software Inspections},
  booktitle = {19th {IEEE} International Conference on Automated Software Engineering
               {(ASE} 2004), 20-25 September 2004, Linz, Austria},
  pages     = {46-55},
  year      = {2004},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/KalinowskiT04.pdf},  
  doi       = {10.1109/ASE.2004.10060},
}

@inproceedings{KalinowskiT04a,
  author    = {Marcos Kalinowski and
               Guilherme Horta Travassos},
  title     = {{ISPIS:} {A} Framework Supporting Software Inspection Processes},
  booktitle = {19th {IEEE} International Conference on Automated Software Engineering
               {(ASE} 2004), 20-25 September 2004, Linz, Austria},
  pages     = {392-393},
  year      = {2004},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/KalinowskiT04a.pdf},
  doi       = {10.1109/ASE.2004.10072},
}

@inproceedings{SpinolaKT04,
  author    = {Rodrigo Oliveira Sp{\'{\i}}nola and
               Marcos Kalinowski and
			   Guilherme Horta Travassos},
  title     = {Uma Infra-Estrutura para Integra{\c{c}}{\~a}o de Ferramentas CASE},
  abstract  = {CASE Tools integration approaches have been developed into the software development environments (SDE) context. Usually, these approaches are based on the SDE canonical model or peer-application needs. Together with the technological evolution described by the technical literature, we hade identified a requirements set to build an innovative integration approach that deals with semantical and s yntactical heterogeneity. Such approach has been implemented by a CASE tool named xMapper. A case study to evaluate its feasibility was also performed. This paper describes xMapper, the case study results and shows xMapper using to integrate real CASE tools into a software inspection process supporting infra-structure.},
  booktitle = {{SBES} 2004 - Proceedings of the Brazilian Symposium on Software Engineering, Brasilia, Brazil},
  pages     = {1-16},
  year      = {2004},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/SpinolaKT04.pdf},
  doi       = {},
}

CASE Tools integration approaches have been developed into the software development environments (SDE) context. Usually, these approaches are based on the SDE canonical model or peer-application needs. Together with the technological evolution described by the technical literature, we hade identified a requirements set to build an innovative integration approach that deals with semantical and s yntactical heterogeneity. Such approach has been implemented by a CASE tool named xMapper. A case study to evaluate its feasibility was also performed. This paper describes xMapper, the case study results and shows xMapper using to integrate real CASE tools into a software inspection process supporting infra-structure.

@inproceedings{KalinowskiST04,
  author    = {Marcos Kalinowski and
               Rodrigo Oliveira Sp{\'{\i}}nola and
			   Guilherme Horta Travassos},
  title     = {Infra-Estrutura Computacional para Apoio ao Processo de Inspe{\c{c}}{\~a}o de Software},
  abstract  = {Software inspections aim to improve software quality by the analysis of software artifacts, detecting and removing their defects before these artifacts can be delivered to the following software life cycle activity. Some knowledge regarding software inspections have been acquired by empirical studies. However, there is no indication that do exist computational support exploring such knowledge. This paper describes a computational framework to support the software inspection process whose requirements set is derived from knowledge acquired by empirical studies.},
  booktitle = {{SBQS} 2004 - Proceedings of the Brazilian Symposium on Software Quality, Brasilia, Brazil},
  pages     = {1-15},
  year      = {2004},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/KalinowskiST04.pdf},
  doi       = {},
}

Software inspections aim to improve software quality by the analysis of software artifacts, detecting and removing their defects before these artifacts can be delivered to the following software life cycle activity. Some knowledge regarding software inspections have been acquired by empirical studies. However, there is no indication that do exist computational support exploring such knowledge. This paper describes a computational framework to support the software inspection process whose requirements set is derived from knowledge acquired by empirical studies.

@inproceedings{ManganAKBW02,
  author    = {Marco Aur{\'{e}}lio Souza Mangan and
               Renata Mendes de Araujo and
               Marcos Kalinowski and
               Marcos R. S. Borges and
               Cl{\'{a}}udia Maria Lima Werner},
  title     = {Towards the Evaluation of Awareness Information Support Applied to
               Peer Reviews of Software Engineering Diagrams},
  booktitle = {Proceedings of the Seventh International Conference on {CSCW} in Design,
               {CSCWD} 2002, September 25-27, 2002, Rio de Janeiro, Brazil},
  pages     = {49-54},
  year      = {2002},
  urlAuthor_version       = {http://www.inf.puc-rio.br/~kalinowski/publications/ManganAKBW02.pdf},
  doi       = {10.1109/CSCWD.2002.1047647},
}