From 1e9833e191c85330c25d089db12e56fbdff21947 Mon Sep 17 00:00:00 2001 From: Deon Griessel Date: Tue, 1 Aug 2017 19:30:46 +0000 Subject: [PATCH] SPOT-212: [Ingest] [Proxy] bluecoat.py parser not able to save into Hive table Replaced saveAsTable syntax (deprecated for Hive tables in Spark 2.0) with insertInto --- spot-ingest/pipelines/proxy/bluecoat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spot-ingest/pipelines/proxy/bluecoat.py b/spot-ingest/pipelines/proxy/bluecoat.py index de9ca6c7..31d89cad 100644 --- a/spot-ingest/pipelines/proxy/bluecoat.py +++ b/spot-ingest/pipelines/proxy/bluecoat.py @@ -121,7 +121,7 @@ def save_data(rdd,sqc,db,db_table,topic): sqc.setConf("hive.exec.dynamic.partition", "true") sqc.setConf("hive.exec.dynamic.partition.mode", "nonstrict") hive_table = "{0}.{1}".format(db,db_table) - df.write.saveAsTable(hive_table,format="parquet",mode="append",partitionBy=('y','m','d','h')) + df.write.format("parquet").mode("append").insertInto(hive_table) else: print("------------------------LISTENING KAFKA TOPIC:{0}------------------------".format(topic))