将 scala 中的伴随对象转换为 Python

Question

我已将以下 Spark Scala 代码翻译成 Python 版本。

package wscalalearning00
import org.apache.spark._
import org.apache.spark.SparkContext._
import org.apache.log4j._
import org.apache.spark.util.StatCounter

object wtry001 {
 def main(args: Array[String]) {

     Logger.getLogger("org").setLevel(Level.ERROR)
   class BballStatCounter extends Serializable {
          val stats: StatCounter = new StatCounter()
          var missing: Long = 0

          def add(x: Double): BballStatCounter = {
             if (x.isNaN) {
                  missing += 1
                } else {

                stats.merge(x)
                }
                this
              }

  }
 object BballStatCounter extends Serializable {
  def apply(x: Double) = new BballStatCounter().add(x)
}

     // Create a SparkContext using every core of the local machine
    val sc = new SparkContext(new SparkConf().setAppName("Spark Word Count").setMaster("local")) 
    val testData = (1 to 10000).toArray.map(x=>x.toDouble)
    val stats1 = sc.parallelize(testData)
    val stat3 = stats1.map(b=>BballStatCounter(b))
    stat3.foreach(println)        

 }
}

Python 版本：

 import math
    import findspark
    findspark.init()
    from pyspark.sql import SparkSession
    from pyspark.statcounter import StatCounter
    class BballStatCounter(object):

        stats = StatCounter()
        missing=0

        @staticmethod
        def add(x):
            print("add")
            if math.isnan(x):
                BballStatCounter.missing += x
            else:
               BballStatCounter.stats.merge(x)
            return BballStatCounter.stats

    conf = SparkConf().setAppName("SparkExampleRDD").setMaster("local")
    sc = SparkContext(conf=conf)
    testData =map(lambda x: float(x),range(0,10000))
    stats1 = sc.parallelize(testData)
    print(stats1)
    stat3 = stats1.map(lambda b: BballStatCounter.add(b))
    stat3.foreach(print)

在执行上述 Python 代码时，它会打印 (count: 22, mean: 10.5, stdev: 6.34428877022, max: 21.0, min: 0.0)，而 Scala 代码会打印类似 BballStatCounter$2@7a811dd5 的内容。我认为 Scala 代码正在返回同一类的实例。如果我的一般方法和/或语法错误，请告诉我。谢谢。

Answer 1

试试这个

class BballStatCounter:

   def __init__(self):
      self.stats = StatCounter()
      self.missing = 0

    @staticmethod
    def add(x):
        stats = StatCounter()
        missing = 0
        if math.isnan(x):
            BballStatCounter.missing += x
        else:
            BballStatCounter.stats.merge(x,stats,missing)


    def merge(other,stats,missing):
       # stats = BballStatCounter.StatCounter()
        #other = BballStatCounter()
        BballStatCounter.stats.merge(other.stats)
        BballStatCounter.missing = +other.missing
        return self

然后做

stat3 = stats1.map(lambda b: BballStatCounter.add(b))
stat3.foreach(print)

我可能不会给你正确的代码，因为我对函数的作用知之甚少。只关注我如何在函数之间传递变量。- 希望你能找到答案。