org.apache.spark.sql.hive

Class HiveUtils

java.lang.Object

org.apache.spark.sql.hive.HiveUtils

public class HiveUtils
extends java.lang.Object

Constructor Summary

Constructors

Constructor and Description
HiveUtils()

Method Summary

Methods

Modifier and Type	Method and Description
static	CONVERT_CTAS()
static	CONVERT_METASTORE_ORC()
static	CONVERT_METASTORE_PARQUET_WITH_SCHEMA_MERGING()
static	CONVERT_METASTORE_PARQUET()
static	HIVE_EXECUTION_VERSION()
static	HIVE_METASTORE_BARRIER_PREFIXES()
static	HIVE_METASTORE_JARS()
static	HIVE_METASTORE_SHARED_PREFIXES()
static	HIVE_METASTORE_VERSION()
static	HIVE_THRIFT_SERVER_ASYNC()
static java.lang.String	hiveExecutionVersion() The version of hive used internally by Spark SQL.
protected static void	initializeLogIfNecessary(boolean isInterpreter)
protected static boolean	isTraceEnabled()
protected static org.slf4j.Logger	log()
protected static void	logDebug(scala.Function0<java.lang.String> msg)
protected static void	logDebug(scala.Function0<java.lang.String> msg, java.lang.Throwable throwable)
protected static void	logError(scala.Function0<java.lang.String> msg)
protected static void	logError(scala.Function0<java.lang.String> msg, java.lang.Throwable throwable)
protected static void	logInfo(scala.Function0<java.lang.String> msg)
protected static void	logInfo(scala.Function0<java.lang.String> msg, java.lang.Throwable throwable)
protected static java.lang.String	logName()
protected static void	logTrace(scala.Function0<java.lang.String> msg)
protected static void	logTrace(scala.Function0<java.lang.String> msg, java.lang.Throwable throwable)
protected static void	logWarning(scala.Function0<java.lang.String> msg)
protected static void	logWarning(scala.Function0<java.lang.String> msg, java.lang.Throwable throwable)
protected static org.apache.spark.sql.hive.client.HiveClientImpl	newClientForExecution(SparkConf conf, org.apache.hadoop.conf.Configuration hadoopConf) Create a HiveClient used for execution.
protected static org.apache.spark.sql.hive.client.HiveClient	newClientForMetadata(SparkConf conf, org.apache.hadoop.conf.Configuration hadoopConf) Create a HiveClient used to retrieve metadata from the Hive MetaStore.
protected static org.apache.spark.sql.hive.client.HiveClient	newClientForMetadata(SparkConf conf, org.apache.hadoop.conf.Configuration hadoopConf, scala.collection.immutable.Map<java.lang.String,java.lang.String> configurations)
static scala.collection.immutable.Map<java.lang.String,java.lang.String>	newTemporaryConfiguration(boolean useInMemoryDerby) Constructs a configuration for hive, where the metastore is located in a temp directory.
protected static scala.collection.Seq<org.apache.spark.sql.types.AtomicType>	primitiveTypes()
protected static java.lang.String	toHiveString(scala.Tuple2<java.lang.Object,DataType> a)
protected static java.lang.String	toHiveStructString(scala.Tuple2<java.lang.Object,DataType> a) Hive outputs fields of structs slightly differently than top level attributes.
static SparkContext	withHiveExternalCatalog(SparkContext sc)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Constructor Detail

HiveUtils

public HiveUtils()

Method Detail

withHiveExternalCatalog

public static SparkContext withHiveExternalCatalog(SparkContext sc)

hiveExecutionVersion

public static java.lang.String hiveExecutionVersion()

The version of hive used internally by Spark SQL.

HIVE_METASTORE_VERSION

public static  HIVE_METASTORE_VERSION()

HIVE_EXECUTION_VERSION

public static  HIVE_EXECUTION_VERSION()

HIVE_METASTORE_JARS

public static  HIVE_METASTORE_JARS()

CONVERT_METASTORE_PARQUET

public static  CONVERT_METASTORE_PARQUET()

CONVERT_METASTORE_PARQUET_WITH_SCHEMA_MERGING

public static  CONVERT_METASTORE_PARQUET_WITH_SCHEMA_MERGING()

CONVERT_CTAS

public static  CONVERT_CTAS()

CONVERT_METASTORE_ORC

public static  CONVERT_METASTORE_ORC()

HIVE_METASTORE_SHARED_PREFIXES

public static  HIVE_METASTORE_SHARED_PREFIXES()

HIVE_METASTORE_BARRIER_PREFIXES

public static  HIVE_METASTORE_BARRIER_PREFIXES()

HIVE_THRIFT_SERVER_ASYNC

public static  HIVE_THRIFT_SERVER_ASYNC()

newClientForExecution

protected static org.apache.spark.sql.hive.client.HiveClientImpl newClientForExecution(SparkConf conf,
                                                                    org.apache.hadoop.conf.Configuration hadoopConf)

Create a HiveClient used for execution.

Currently this must always be Hive 13 as this is the version of Hive that is packaged with Spark SQL. This copy of the client is used for execution related tasks like registering temporary functions or ensuring that the ThreadLocal SessionState is correctly populated. This copy of Hive is *not* used for storing persistent metadata, and only point to a dummy metastore in a temporary directory.

Parameters:

conf - (undocumented)

hadoopConf - (undocumented)

Returns:

(undocumented)

newClientForMetadata

protected static org.apache.spark.sql.hive.client.HiveClient newClientForMetadata(SparkConf conf,
                                                               org.apache.hadoop.conf.Configuration hadoopConf)

Create a HiveClient used to retrieve metadata from the Hive MetaStore.

The version of the Hive client that is used here must match the metastore that is configured in the hive-site.xml file.

Parameters:

conf - (undocumented)

hadoopConf - (undocumented)

Returns:

(undocumented)

newClientForMetadata

protected static org.apache.spark.sql.hive.client.HiveClient newClientForMetadata(SparkConf conf,
                                                               org.apache.hadoop.conf.Configuration hadoopConf,
                                                               scala.collection.immutable.Map<java.lang.String,java.lang.String> configurations)

newTemporaryConfiguration

public static scala.collection.immutable.Map<java.lang.String,java.lang.String> newTemporaryConfiguration(boolean useInMemoryDerby)

Constructs a configuration for hive, where the metastore is located in a temp directory.

primitiveTypes

protected static scala.collection.Seq<org.apache.spark.sql.types.AtomicType> primitiveTypes()

toHiveString

protected static java.lang.String toHiveString(scala.Tuple2<java.lang.Object,DataType> a)

toHiveStructString

protected static java.lang.String toHiveStructString(scala.Tuple2<java.lang.Object,DataType> a)

Hive outputs fields of structs slightly differently than top level attributes.

logName

protected static java.lang.String logName()

log

protected static org.slf4j.Logger log()

logInfo

protected static void logInfo(scala.Function0<java.lang.String> msg)

logDebug

protected static void logDebug(scala.Function0<java.lang.String> msg)

logTrace

protected static void logTrace(scala.Function0<java.lang.String> msg)

logWarning

protected static void logWarning(scala.Function0<java.lang.String> msg)

logError

protected static void logError(scala.Function0<java.lang.String> msg)

logInfo

protected static void logInfo(scala.Function0<java.lang.String> msg,
           java.lang.Throwable throwable)

logDebug

protected static void logDebug(scala.Function0<java.lang.String> msg,
            java.lang.Throwable throwable)

logTrace

protected static void logTrace(scala.Function0<java.lang.String> msg,
            java.lang.Throwable throwable)

logWarning

protected static void logWarning(scala.Function0<java.lang.String> msg,
              java.lang.Throwable throwable)

logError

protected static void logError(scala.Function0<java.lang.String> msg,
            java.lang.Throwable throwable)

isTraceEnabled

protected static boolean isTraceEnabled()

initializeLogIfNecessary

protected static void initializeLogIfNecessary(boolean isInterpreter)