Big Data, MapReduce, Hadoop, and Spark with Python

此书不错，很短，且想打通PYTHON和大数据架构的关系。

先看一次，计划把这个文档作个翻译。

先来一个模拟MAPREDUCE的东东。。。

mapper.py

class Mapper:
    def map(self, data):
        returnval = []
        counts = {}
        for line in data:
            words = line.split()
            for w in words:
                counts[w] = counts.get(w, 0) + 1
        for w, c in counts.iteritems():
            returnval.append((w, c))
        print "Mapper result:"
        print returnval
        return returnval
    

 

reducer.py

class Reducer:
    def reduce(self, d):
        returnval = []
        for k, v in d.iteritems():
            returnval.append("%s\t%s"%(k, sum(v)))
        print "Reducer result:"
        print returnval
        return returnval

 

main.py

from mapper import Mapper
from reducer import Reducer

class JobRunner:
    def run(self, Mapper, Reducer, data):
        # map
        mapper = Mapper()
        tuples = mapper.map(data)

        # combine
        combined = {}
        for k, v in tuples:
            if k not in combined:
                combined[k] = []
            combined[k].append(v)
        print "combined result:"
        print combined

        # reduce
        reducer = Reducer()
        output = reducer.reduce(combined)

        # do something with output
        for line in output:
            print line

runner = JobRunner()
runner.run(Mapper, Reducer, open("input.txt"))