Since for me this problem presented mainly on iOS, I provide the code to fix that only on iOS:
if(!!navigator.platform && /iPad|iPhone|iPod/.test(navigator.platform)) {
var $modalRep = $('#modal-id'),
startScrollY = null,
moveDiv;
$modalRep.on('touchmove', function(ev) {
ev.preventDefault();
moveDiv = startScrollY - ev.touches[0].clientY;
startScrollY = ev.touches[0].clientY;
var el = $(ev.target).parents('#div-that-scrolls');
// #div-that-scrolls is a child of #modal-id
el.scrollTop(el.scrollTop() + moveDiv);
});
$modalRep.on('touchstart', function(ev) {
startScrollY = ev.touches[0].clientY;
});
}