如何将字符串 json 转换为 scala 数据帧？

Question

我有一个字符串 json，我正在尝试创建一个数据框。使用下面的代码我可以实现，但是我正在构建jar的环境，它抱怨toDS。还有其他方法可以实现吗？

val abc="""[{"orders":{"order_id":{"path":"orderid","type":"string"},"customer_id":{"path":"customers.customerId","type":"string"},"offer_id":{"path":"Offers.Offerid","type":"string"}},"products":{"product_id":{"path":"product_id","type":"string"},"product_name":{"path":"products.productname","type":"string"}}}]"""

val df = spark.read.json(Seq(abc).toDS());
df.show()

Answer 1

您可以使用

import spark.implicits._

导入 Spark 隐式，它应该可以工作。

scala> spark
res23: org.apache.spark.sql.SparkSession = org.apache.spark.sql.SparkSession@8373d56

scala> import spark.implicits._
import spark.implicits._

scala> val abc="""[{"orders":{"order_id":{"path":"orderid","type":"string"},"customer_id":{"path":"customers.customerId","type":"string"},"offer_id":{"path":"Offers.Offerid","type":"string"}},"products":{"product_id":{"path":"product_id","type":"string"},"product_name":{"path":"products.productname","type":"string"}}}]"""
abc: String = [{"orders":{"order_id":{"path":"orderid","type":"string"},"customer_id":{"path":"customers.customerId","type":"string"},"offer_id":{"path":"Offers.Offerid","type":"string"}},"products":{"product_id":{"path":"product_id","type":"string"},"product_name":{"path":"products.productname","type":"string"}}}]

scala> val df = spark.read.json(Seq(abc).toDS())
df: org.apache.spark.sql.DataFrame = [orders: struct<customer_id: struct<path: string, type: string>, offer_id: struct<path: string, type: string> ... 1 more field>, products: struct<product_id: struct<path: string, type: string>, product_name: struct<path: string, type: string>>]

scala> df.show(false)
+-----------------------------------------------------------------------------+------------------------------------------------------+
|orders                                                                       |products                                              |
+-----------------------------------------------------------------------------+------------------------------------------------------+
|{{customers.customerId, string}, {Offers.Offerid, string}, {orderid, string}}|{{product_id, string}, {products.productname, string}}|
+-----------------------------------------------------------------------------+------------------------------------------------------+