mapPartitions

rdd.mapPartitions{ partition =>
if (!partition.isEmpty) {
// Some setup code here
partition.map(item => {
val output =yourfunction(item)
if (!partition.hasNext){
// Some cleanup code here
}
output
})
} else {
// return an empty Iterator of your return type
}
}

 

mapPartitions IO等资源管理