我试图从数据集中替换空值,而数据集是以列表的形式存在的。我使用了下面的代码,但还是没有得到我想要的结果。我应该怎么做?
我正在使用下面的代码。
val mergedDS =customerDS.join(acctstep1,Seq("customerId"),"outer")
.withColumn("numberAccounts", 'numberAccounts.cast("Int"))
.withColumn("totalBalance",'totalBalance.cast("Long"))
// Lets remove all the accounts with missing values for customers
val customerAccountOutputDS = mergedDS.as[CustomerAccountOutput].na.fill(0).show(false)
print(customerAccountOutputDS)
import org.apache.spark.sql.functions._
import org.apache.spark.sql.SparkSession;
object DefaultEmptyArray {
def main(args: Array[String]): Unit = {
val input = List(Bean(List("A","B")),Bean(null),Bean(List("C","D")))
val spark = SparkSession.builder().master("local[*]").getOrCreate();
val df = spark.createDataFrame(input)
df.select("inputList")
.withColumn("outputList",when(col("inputList").isNull,Array[String]()).otherwise(col("inputList")))
.show()
}
}
case class Bean( inputList : List[String])