public class functions
extends Object
Constructor and Description |
---|
functions() |
Modifier and Type | Method and Description |
---|---|
static Column |
abs(Column e)
Computes the absolute value.
|
static Column |
acos(Column e)
Computes the cosine inverse of the given value; the returned angle is in the range
0.0 through pi.
|
static Column |
acos(String columnName)
Computes the cosine inverse of the given column; the returned angle is in the range
0.0 through pi.
|
static Column |
approxCountDistinct(Column e)
Aggregate function: returns the approximate number of distinct items in a group.
|
static Column |
approxCountDistinct(Column e,
double rsd)
Aggregate function: returns the approximate number of distinct items in a group.
|
static Column |
approxCountDistinct(String columnName)
Aggregate function: returns the approximate number of distinct items in a group.
|
static Column |
approxCountDistinct(String columnName,
double rsd)
Aggregate function: returns the approximate number of distinct items in a group.
|
static Column |
array(Column... cols)
Creates a new array column.
|
static Column |
array(scala.collection.Seq<Column> cols)
Creates a new array column.
|
static Column |
array(String colName,
scala.collection.Seq<String> colNames)
Creates a new array column.
|
static Column |
asc(String columnName)
Returns a sort expression based on ascending order of the column.
|
static Column |
asin(Column e)
Computes the sine inverse of the given value; the returned angle is in the range
-pi/2 through pi/2.
|
static Column |
asin(String columnName)
Computes the sine inverse of the given column; the returned angle is in the range
-pi/2 through pi/2.
|
static Column |
atan(Column e)
Computes the tangent inverse of the given value.
|
static Column |
atan(String columnName)
Computes the tangent inverse of the given column.
|
static Column |
atan2(Column l,
Column r)
Returns the angle theta from the conversion of rectangular coordinates (x, y) to
polar coordinates (r, theta).
|
static Column |
atan2(Column l,
double r)
Returns the angle theta from the conversion of rectangular coordinates (x, y) to
polar coordinates (r, theta).
|
static Column |
atan2(Column l,
String rightName)
Returns the angle theta from the conversion of rectangular coordinates (x, y) to
polar coordinates (r, theta).
|
static Column |
atan2(double l,
Column r)
Returns the angle theta from the conversion of rectangular coordinates (x, y) to
polar coordinates (r, theta).
|
static Column |
atan2(double l,
String rightName)
Returns the angle theta from the conversion of rectangular coordinates (x, y) to
polar coordinates (r, theta).
|
static Column |
atan2(String leftName,
Column r)
Returns the angle theta from the conversion of rectangular coordinates (x, y) to
polar coordinates (r, theta).
|
static Column |
atan2(String leftName,
double r)
Returns the angle theta from the conversion of rectangular coordinates (x, y) to
polar coordinates (r, theta).
|
static Column |
atan2(String leftName,
String rightName)
Returns the angle theta from the conversion of rectangular coordinates (x, y) to
polar coordinates (r, theta).
|
static Column |
avg(Column e)
Aggregate function: returns the average of the values in a group.
|
static Column |
avg(String columnName)
Aggregate function: returns the average of the values in a group.
|
static Column |
bitwiseNOT(Column e)
Computes bitwise NOT.
|
static Column |
callUDF(scala.Function0<?> f,
DataType returnType)
Call a Scala function of 0 arguments as user-defined function (UDF).
|
static Column |
callUDF(scala.Function1<?,?> f,
DataType returnType,
Column arg1)
Call a Scala function of 1 arguments as user-defined function (UDF).
|
static Column |
callUDF(scala.Function10<?,?,?,?,?,?,?,?,?,?,?> f,
DataType returnType,
Column arg1,
Column arg2,
Column arg3,
Column arg4,
Column arg5,
Column arg6,
Column arg7,
Column arg8,
Column arg9,
Column arg10)
Call a Scala function of 10 arguments as user-defined function (UDF).
|
static Column |
callUDF(scala.Function2<?,?,?> f,
DataType returnType,
Column arg1,
Column arg2)
Call a Scala function of 2 arguments as user-defined function (UDF).
|
static Column |
callUDF(scala.Function3<?,?,?,?> f,
DataType returnType,
Column arg1,
Column arg2,
Column arg3)
Call a Scala function of 3 arguments as user-defined function (UDF).
|
static Column |
callUDF(scala.Function4<?,?,?,?,?> f,
DataType returnType,
Column arg1,
Column arg2,
Column arg3,
Column arg4)
Call a Scala function of 4 arguments as user-defined function (UDF).
|
static Column |
callUDF(scala.Function5<?,?,?,?,?,?> f,
DataType returnType,
Column arg1,
Column arg2,
Column arg3,
Column arg4,
Column arg5)
Call a Scala function of 5 arguments as user-defined function (UDF).
|
static Column |
callUDF(scala.Function6<?,?,?,?,?,?,?> f,
DataType returnType,
Column arg1,
Column arg2,
Column arg3,
Column arg4,
Column arg5,
Column arg6)
Call a Scala function of 6 arguments as user-defined function (UDF).
|
static Column |
callUDF(scala.Function7<?,?,?,?,?,?,?,?> f,
DataType returnType,
Column arg1,
Column arg2,
Column arg3,
Column arg4,
Column arg5,
Column arg6,
Column arg7)
Call a Scala function of 7 arguments as user-defined function (UDF).
|
static Column |
callUDF(scala.Function8<?,?,?,?,?,?,?,?,?> f,
DataType returnType,
Column arg1,
Column arg2,
Column arg3,
Column arg4,
Column arg5,
Column arg6,
Column arg7,
Column arg8)
Call a Scala function of 8 arguments as user-defined function (UDF).
|
static Column |
callUDF(scala.Function9<?,?,?,?,?,?,?,?,?,?> f,
DataType returnType,
Column arg1,
Column arg2,
Column arg3,
Column arg4,
Column arg5,
Column arg6,
Column arg7,
Column arg8,
Column arg9)
Call a Scala function of 9 arguments as user-defined function (UDF).
|
static Column |
callUdf(String udfName,
scala.collection.Seq<Column> cols)
Call an user-defined function.
|
static Column |
cbrt(Column e)
Computes the cube-root of the given value.
|
static Column |
cbrt(String columnName)
Computes the cube-root of the given column.
|
static Column |
ceil(Column e)
Computes the ceiling of the given value.
|
static Column |
ceil(String columnName)
Computes the ceiling of the given column.
|
static Column |
coalesce(Column... e)
Returns the first column that is not null.
|
static Column |
coalesce(scala.collection.Seq<Column> e)
Returns the first column that is not null.
|
static Column |
col(String colName)
Returns a
Column based on the given column name. |
static Column |
column(String colName)
Returns a
Column based on the given column name. |
static Column |
cos(Column e)
Computes the cosine of the given value.
|
static Column |
cos(String columnName)
Computes the cosine of the given column.
|
static Column |
cosh(Column e)
Computes the hyperbolic cosine of the given value.
|
static Column |
cosh(String columnName)
Computes the hyperbolic cosine of the given column.
|
static Column |
count(Column e)
Aggregate function: returns the number of items in a group.
|
static Column |
count(String columnName)
Aggregate function: returns the number of items in a group.
|
static Column |
countDistinct(Column expr,
Column... exprs)
Aggregate function: returns the number of distinct items in a group.
|
static Column |
countDistinct(Column expr,
scala.collection.Seq<Column> exprs)
Aggregate function: returns the number of distinct items in a group.
|
static Column |
countDistinct(String columnName,
scala.collection.Seq<String> columnNames)
Aggregate function: returns the number of distinct items in a group.
|
static Column |
countDistinct(String columnName,
String... columnNames)
Aggregate function: returns the number of distinct items in a group.
|
static Column |
cumeDist()
Window function: returns the cumulative distribution of values within a window partition,
i.e.
|
static Column |
denseRank()
Window function: returns the rank of rows within a window partition, without any gaps.
|
static Column |
desc(String columnName)
Returns a sort expression based on the descending order of the column.
|
static Column |
exp(Column e)
Computes the exponential of the given value.
|
static Column |
exp(String columnName)
Computes the exponential of the given column.
|
static Column |
explode(Column e)
Creates a new row for each element in the given array or map column.
|
static Column |
expm1(Column e)
Computes the exponential of the given value minus one.
|
static Column |
expm1(String columnName)
Computes the exponential of the given column.
|
static Column |
first(Column e)
Aggregate function: returns the first value in a group.
|
static Column |
first(String columnName)
Aggregate function: returns the first value of a column in a group.
|
static Column |
floor(Column e)
Computes the floor of the given value.
|
static Column |
floor(String columnName)
Computes the floor of the given column.
|
static Column |
hypot(Column l,
Column r)
Computes
sqrt(a^2^ + b^2^) without intermediate overflow or underflow. |
static Column |
hypot(Column l,
double r)
Computes
sqrt(a^2^ + b^2^) without intermediate overflow or underflow. |
static Column |
hypot(Column l,
String rightName)
Computes
sqrt(a^2^ + b^2^) without intermediate overflow or underflow. |
static Column |
hypot(double l,
Column r)
Computes
sqrt(a^2^ + b^2^) without intermediate overflow or underflow. |
static Column |
hypot(double l,
String rightName)
Computes
sqrt(a^2^ + b^2^) without intermediate overflow or underflow. |
static Column |
hypot(String leftName,
Column r)
Computes
sqrt(a^2^ + b^2^) without intermediate overflow or underflow. |
static Column |
hypot(String leftName,
double r)
Computes
sqrt(a^2^ + b^2^) without intermediate overflow or underflow. |
static Column |
hypot(String leftName,
String rightName)
Computes
sqrt(a^2^ + b^2^) without intermediate overflow or underflow. |
static Column |
lag(Column e,
int offset)
Window function: returns the value that is
offset rows before the current row, and
null if there is less than offset rows before the current row. |
static Column |
lag(Column e,
int offset,
Object defaultValue)
Window function: returns the value that is
offset rows before the current row, and
defaultValue if there is less than offset rows before the current row. |
static Column |
lag(String columnName,
int offset)
Window function: returns the value that is
offset rows before the current row, and
null if there is less than offset rows before the current row. |
static Column |
lag(String columnName,
int offset,
Object defaultValue)
Window function: returns the value that is
offset rows before the current row, and
defaultValue if there is less than offset rows before the current row. |
static Column |
last(Column e)
Aggregate function: returns the last value in a group.
|
static Column |
last(String columnName)
Aggregate function: returns the last value of the column in a group.
|
static Column |
lead(Column e,
int offset)
Window function: returns the value that is
offset rows after the current row, and
null if there is less than offset rows after the current row. |
static Column |
lead(Column e,
int offset,
Object defaultValue)
Window function: returns the value that is
offset rows after the current row, and
defaultValue if there is less than offset rows after the current row. |
static Column |
lead(String columnName,
int offset)
Window function: returns the value that is
offset rows after the current row, and
null if there is less than offset rows after the current row. |
static Column |
lead(String columnName,
int offset,
Object defaultValue)
Window function: returns the value that is
offset rows after the current row, and
defaultValue if there is less than offset rows after the current row. |
static Column |
lit(Object literal)
Creates a
Column of literal value. |
static Column |
log(Column e)
Computes the natural logarithm of the given value.
|
static Column |
log(String columnName)
Computes the natural logarithm of the given column.
|
static Column |
log10(Column e)
Computes the logarithm of the given value in Base 10.
|
static Column |
log10(String columnName)
Computes the logarithm of the given value in Base 10.
|
static Column |
log1p(Column e)
Computes the natural logarithm of the given value plus one.
|
static Column |
log1p(String columnName)
Computes the natural logarithm of the given column plus one.
|
static Column |
lower(Column e)
Converts a string exprsesion to lower case.
|
static Column |
max(Column e)
Aggregate function: returns the maximum value of the expression in a group.
|
static Column |
max(String columnName)
Aggregate function: returns the maximum value of the column in a group.
|
static Column |
mean(Column e)
Aggregate function: returns the average of the values in a group.
|
static Column |
mean(String columnName)
Aggregate function: returns the average of the values in a group.
|
static Column |
min(Column e)
Aggregate function: returns the minimum value of the expression in a group.
|
static Column |
min(String columnName)
Aggregate function: returns the minimum value of the column in a group.
|
static Column |
monotonicallyIncreasingId()
A column expression that generates monotonically increasing 64-bit integers.
|
static Column |
negate(Column e)
Unary minus, i.e.
|
static Column |
not(Column e)
Inversion of boolean expression, i.e.
|
static Column |
ntile(int n)
Window function: returns the ntile group id (from 1 to
n inclusive) in an ordered window
partition. |
static Column |
percentRank()
Window function: returns the relative rank (i.e.
|
static Column |
pow(Column l,
Column r)
Returns the value of the first argument raised to the power of the second argument.
|
static Column |
pow(Column l,
double r)
Returns the value of the first argument raised to the power of the second argument.
|
static Column |
pow(Column l,
String rightName)
Returns the value of the first argument raised to the power of the second argument.
|
static Column |
pow(double l,
Column r)
Returns the value of the first argument raised to the power of the second argument.
|
static Column |
pow(double l,
String rightName)
Returns the value of the first argument raised to the power of the second argument.
|
static Column |
pow(String leftName,
Column r)
Returns the value of the first argument raised to the power of the second argument.
|
static Column |
pow(String leftName,
double r)
Returns the value of the first argument raised to the power of the second argument.
|
static Column |
pow(String leftName,
String rightName)
Returns the value of the first argument raised to the power of the second argument.
|
static Column |
rand()
Generate a random column with i.i.d.
|
static Column |
rand(long seed)
Generate a random column with i.i.d.
|
static Column |
randn()
Generate a column with i.i.d.
|
static Column |
randn(long seed)
Generate a column with i.i.d.
|
static Column |
rank()
Window function: returns the rank of rows within a window partition.
|
static Column |
rint(Column e)
Returns the double value that is closest in value to the argument and
is equal to a mathematical integer.
|
static Column |
rint(String columnName)
Returns the double value that is closest in value to the argument and
is equal to a mathematical integer.
|
static Column |
rowNumber()
Window function: returns a sequential number starting at 1 within a window partition.
|
static Column |
signum(Column e)
Computes the signum of the given value.
|
static Column |
signum(String columnName)
Computes the signum of the given column.
|
static Column |
sin(Column e)
Computes the sine of the given value.
|
static Column |
sin(String columnName)
Computes the sine of the given column.
|
static Column |
sinh(Column e)
Computes the hyperbolic sine of the given value.
|
static Column |
sinh(String columnName)
Computes the hyperbolic sine of the given column.
|
static Column |
sparkPartitionId()
Partition ID of the Spark task.
|
static Column |
sqrt(Column e)
Computes the square root of the specified float value.
|
static Column |
struct(Column... cols)
Creates a new struct column.
|
static Column |
struct(scala.collection.Seq<Column> cols)
Creates a new struct column.
|
static Column |
struct(String colName,
scala.collection.Seq<String> colNames)
Creates a new struct column that composes multiple input columns.
|
static Column |
sum(Column e)
Aggregate function: returns the sum of all values in the expression.
|
static Column |
sum(String columnName)
Aggregate function: returns the sum of all values in the given column.
|
static Column |
sumDistinct(Column e)
Aggregate function: returns the sum of distinct values in the expression.
|
static Column |
sumDistinct(String columnName)
Aggregate function: returns the sum of distinct values in the expression.
|
static Column |
tan(Column e)
Computes the tangent of the given value.
|
static Column |
tan(String columnName)
Computes the tangent of the given column.
|
static Column |
tanh(Column e)
Computes the hyperbolic tangent of the given value.
|
static Column |
tanh(String columnName)
Computes the hyperbolic tangent of the given column.
|
static Column |
toDegrees(Column e)
Converts an angle measured in radians to an approximately equivalent angle measured in degrees.
|
static Column |
toDegrees(String columnName)
Converts an angle measured in radians to an approximately equivalent angle measured in degrees.
|
static Column |
toRadians(Column e)
Converts an angle measured in degrees to an approximately equivalent angle measured in radians.
|
static Column |
toRadians(String columnName)
Converts an angle measured in degrees to an approximately equivalent angle measured in radians.
|
static <RT> UserDefinedFunction |
udf(scala.Function0<RT> f,
scala.reflect.api.TypeTags.TypeTag<RT> evidence$1)
Defines a user-defined function of 0 arguments as user-defined function (UDF).
|
static <RT,A1> UserDefinedFunction |
udf(scala.Function1<A1,RT> f,
scala.reflect.api.TypeTags.TypeTag<RT> evidence$2,
scala.reflect.api.TypeTags.TypeTag<A1> evidence$3)
Defines a user-defined function of 1 arguments as user-defined function (UDF).
|
static <RT,A1,A2,A3,A4,A5,A6,A7,A8,A9,A10> |
udf(scala.Function10<A1,A2,A3,A4,A5,A6,A7,A8,A9,A10,RT> f,
scala.reflect.api.TypeTags.TypeTag<RT> evidence$56,
scala.reflect.api.TypeTags.TypeTag<A1> evidence$57,
scala.reflect.api.TypeTags.TypeTag<A2> evidence$58,
scala.reflect.api.TypeTags.TypeTag<A3> evidence$59,
scala.reflect.api.TypeTags.TypeTag<A4> evidence$60,
scala.reflect.api.TypeTags.TypeTag<A5> evidence$61,
scala.reflect.api.TypeTags.TypeTag<A6> evidence$62,
scala.reflect.api.TypeTags.TypeTag<A7> evidence$63,
scala.reflect.api.TypeTags.TypeTag<A8> evidence$64,
scala.reflect.api.TypeTags.TypeTag<A9> evidence$65,
scala.reflect.api.TypeTags.TypeTag<A10> evidence$66)
Defines a user-defined function of 10 arguments as user-defined function (UDF).
|
static <RT,A1,A2> UserDefinedFunction |
udf(scala.Function2<A1,A2,RT> f,
scala.reflect.api.TypeTags.TypeTag<RT> evidence$4,
scala.reflect.api.TypeTags.TypeTag<A1> evidence$5,
scala.reflect.api.TypeTags.TypeTag<A2> evidence$6)
Defines a user-defined function of 2 arguments as user-defined function (UDF).
|
static <RT,A1,A2,A3> |
udf(scala.Function3<A1,A2,A3,RT> f,
scala.reflect.api.TypeTags.TypeTag<RT> evidence$7,
scala.reflect.api.TypeTags.TypeTag<A1> evidence$8,
scala.reflect.api.TypeTags.TypeTag<A2> evidence$9,
scala.reflect.api.TypeTags.TypeTag<A3> evidence$10)
Defines a user-defined function of 3 arguments as user-defined function (UDF).
|
static <RT,A1,A2,A3,A4> |
udf(scala.Function4<A1,A2,A3,A4,RT> f,
scala.reflect.api.TypeTags.TypeTag<RT> evidence$11,
scala.reflect.api.TypeTags.TypeTag<A1> evidence$12,
scala.reflect.api.TypeTags.TypeTag<A2> evidence$13,
scala.reflect.api.TypeTags.TypeTag<A3> evidence$14,
scala.reflect.api.TypeTags.TypeTag<A4> evidence$15)
Defines a user-defined function of 4 arguments as user-defined function (UDF).
|
static <RT,A1,A2,A3,A4,A5> |
udf(scala.Function5<A1,A2,A3,A4,A5,RT> f,
scala.reflect.api.TypeTags.TypeTag<RT> evidence$16,
scala.reflect.api.TypeTags.TypeTag<A1> evidence$17,
scala.reflect.api.TypeTags.TypeTag<A2> evidence$18,
scala.reflect.api.TypeTags.TypeTag<A3> evidence$19,
scala.reflect.api.TypeTags.TypeTag<A4> evidence$20,
scala.reflect.api.TypeTags.TypeTag<A5> evidence$21)
Defines a user-defined function of 5 arguments as user-defined function (UDF).
|
static <RT,A1,A2,A3,A4,A5,A6> |
udf(scala.Function6<A1,A2,A3,A4,A5,A6,RT> f,
scala.reflect.api.TypeTags.TypeTag<RT> evidence$22,
scala.reflect.api.TypeTags.TypeTag<A1> evidence$23,
scala.reflect.api.TypeTags.TypeTag<A2> evidence$24,
scala.reflect.api.TypeTags.TypeTag<A3> evidence$25,
scala.reflect.api.TypeTags.TypeTag<A4> evidence$26,
scala.reflect.api.TypeTags.TypeTag<A5> evidence$27,
scala.reflect.api.TypeTags.TypeTag<A6> evidence$28)
Defines a user-defined function of 6 arguments as user-defined function (UDF).
|
static <RT,A1,A2,A3,A4,A5,A6,A7> |
udf(scala.Function7<A1,A2,A3,A4,A5,A6,A7,RT> f,
scala.reflect.api.TypeTags.TypeTag<RT> evidence$29,
scala.reflect.api.TypeTags.TypeTag<A1> evidence$30,
scala.reflect.api.TypeTags.TypeTag<A2> evidence$31,
scala.reflect.api.TypeTags.TypeTag<A3> evidence$32,
scala.reflect.api.TypeTags.TypeTag<A4> evidence$33,
scala.reflect.api.TypeTags.TypeTag<A5> evidence$34,
scala.reflect.api.TypeTags.TypeTag<A6> evidence$35,
scala.reflect.api.TypeTags.TypeTag<A7> evidence$36)
Defines a user-defined function of 7 arguments as user-defined function (UDF).
|
static <RT,A1,A2,A3,A4,A5,A6,A7,A8> |
udf(scala.Function8<A1,A2,A3,A4,A5,A6,A7,A8,RT> f,
scala.reflect.api.TypeTags.TypeTag<RT> evidence$37,
scala.reflect.api.TypeTags.TypeTag<A1> evidence$38,
scala.reflect.api.TypeTags.TypeTag<A2> evidence$39,
scala.reflect.api.TypeTags.TypeTag<A3> evidence$40,
scala.reflect.api.TypeTags.TypeTag<A4> evidence$41,
scala.reflect.api.TypeTags.TypeTag<A5> evidence$42,
scala.reflect.api.TypeTags.TypeTag<A6> evidence$43,
scala.reflect.api.TypeTags.TypeTag<A7> evidence$44,
scala.reflect.api.TypeTags.TypeTag<A8> evidence$45)
Defines a user-defined function of 8 arguments as user-defined function (UDF).
|
static <RT,A1,A2,A3,A4,A5,A6,A7,A8,A9> |
udf(scala.Function9<A1,A2,A3,A4,A5,A6,A7,A8,A9,RT> f,
scala.reflect.api.TypeTags.TypeTag<RT> evidence$46,
scala.reflect.api.TypeTags.TypeTag<A1> evidence$47,
scala.reflect.api.TypeTags.TypeTag<A2> evidence$48,
scala.reflect.api.TypeTags.TypeTag<A3> evidence$49,
scala.reflect.api.TypeTags.TypeTag<A4> evidence$50,
scala.reflect.api.TypeTags.TypeTag<A5> evidence$51,
scala.reflect.api.TypeTags.TypeTag<A6> evidence$52,
scala.reflect.api.TypeTags.TypeTag<A7> evidence$53,
scala.reflect.api.TypeTags.TypeTag<A8> evidence$54,
scala.reflect.api.TypeTags.TypeTag<A9> evidence$55)
Defines a user-defined function of 9 arguments as user-defined function (UDF).
|
static Column |
upper(Column e)
Converts a string expression to upper case.
|
static Column |
when(Column condition,
Object value)
Evaluates a list of conditions and returns one of multiple possible result expressions.
|
public static Column countDistinct(Column expr, Column... exprs)
expr
- (undocumented)exprs
- (undocumented)public static Column countDistinct(String columnName, String... columnNames)
columnName
- (undocumented)columnNames
- (undocumented)public static Column array(Column... cols)
cols
- (undocumented)public static Column coalesce(Column... e)
df.select(coalesce(df("a"), df("b")))
e
- (undocumented)public static Column struct(Column... cols)
DataFrame
, or
a derived column expression that is named (i.e. aliased).
cols
- (undocumented)public static Column col(String colName)
Column
based on the given column name.
colName
- (undocumented)public static Column column(String colName)
Column
based on the given column name. Alias of col
.
colName
- (undocumented)public static Column lit(Object literal)
Column
of literal value.
The passed in object is returned directly if it is already a Column
.
If the object is a Scala Symbol, it is converted into a Column
also.
Otherwise, a new Column
is created to represent the literal value.
literal
- (undocumented)public static Column asc(String columnName)
// Sort by dept in ascending order, and then age in descending order.
df.sort(asc("dept"), desc("age"))
columnName
- (undocumented)public static Column desc(String columnName)
// Sort by dept in ascending order, and then age in descending order.
df.sort(asc("dept"), desc("age"))
columnName
- (undocumented)public static Column sum(Column e)
e
- (undocumented)public static Column sum(String columnName)
columnName
- (undocumented)public static Column sumDistinct(Column e)
e
- (undocumented)public static Column sumDistinct(String columnName)
columnName
- (undocumented)public static Column count(Column e)
e
- (undocumented)public static Column count(String columnName)
columnName
- (undocumented)public static Column countDistinct(Column expr, scala.collection.Seq<Column> exprs)
expr
- (undocumented)exprs
- (undocumented)public static Column countDistinct(String columnName, scala.collection.Seq<String> columnNames)
columnName
- (undocumented)columnNames
- (undocumented)public static Column approxCountDistinct(Column e)
e
- (undocumented)public static Column approxCountDistinct(String columnName)
columnName
- (undocumented)public static Column approxCountDistinct(Column e, double rsd)
e
- (undocumented)rsd
- (undocumented)public static Column approxCountDistinct(String columnName, double rsd)
columnName
- (undocumented)rsd
- (undocumented)public static Column avg(Column e)
e
- (undocumented)public static Column avg(String columnName)
columnName
- (undocumented)public static Column first(Column e)
e
- (undocumented)public static Column first(String columnName)
columnName
- (undocumented)public static Column last(Column e)
e
- (undocumented)public static Column last(String columnName)
columnName
- (undocumented)public static Column mean(Column e)
e
- (undocumented)public static Column mean(String columnName)
columnName
- (undocumented)public static Column min(Column e)
e
- (undocumented)public static Column min(String columnName)
columnName
- (undocumented)public static Column max(Column e)
e
- (undocumented)public static Column max(String columnName)
columnName
- (undocumented)public static Column lag(Column e, int offset)
offset
rows before the current row, and
null
if there is less than offset
rows before the current row. For example,
an offset
of one will return the previous row at any given point in the window partition.
This is equivalent to the LAG function in SQL.
e
- (undocumented)offset
- (undocumented)public static Column lag(String columnName, int offset)
offset
rows before the current row, and
null
if there is less than offset
rows before the current row. For example,
an offset
of one will return the previous row at any given point in the window partition.
This is equivalent to the LAG function in SQL.
columnName
- (undocumented)offset
- (undocumented)public static Column lag(String columnName, int offset, Object defaultValue)
offset
rows before the current row, and
defaultValue
if there is less than offset
rows before the current row. For example,
an offset
of one will return the previous row at any given point in the window partition.
This is equivalent to the LAG function in SQL.
columnName
- (undocumented)offset
- (undocumented)defaultValue
- (undocumented)public static Column lag(Column e, int offset, Object defaultValue)
offset
rows before the current row, and
defaultValue
if there is less than offset
rows before the current row. For example,
an offset
of one will return the previous row at any given point in the window partition.
This is equivalent to the LAG function in SQL.
e
- (undocumented)offset
- (undocumented)defaultValue
- (undocumented)public static Column lead(String columnName, int offset)
offset
rows after the current row, and
null
if there is less than offset
rows after the current row. For example,
an offset
of one will return the next row at any given point in the window partition.
This is equivalent to the LEAD function in SQL.
columnName
- (undocumented)offset
- (undocumented)public static Column lead(Column e, int offset)
offset
rows after the current row, and
null
if there is less than offset
rows after the current row. For example,
an offset
of one will return the next row at any given point in the window partition.
This is equivalent to the LEAD function in SQL.
e
- (undocumented)offset
- (undocumented)public static Column lead(String columnName, int offset, Object defaultValue)
offset
rows after the current row, and
defaultValue
if there is less than offset
rows after the current row. For example,
an offset
of one will return the next row at any given point in the window partition.
This is equivalent to the LEAD function in SQL.
columnName
- (undocumented)offset
- (undocumented)defaultValue
- (undocumented)public static Column lead(Column e, int offset, Object defaultValue)
offset
rows after the current row, and
defaultValue
if there is less than offset
rows after the current row. For example,
an offset
of one will return the next row at any given point in the window partition.
This is equivalent to the LEAD function in SQL.
e
- (undocumented)offset
- (undocumented)defaultValue
- (undocumented)public static Column ntile(int n)
n
inclusive) in an ordered window
partition. Fow example, if n
is 4, the first quarter of the rows will get value 1, the second
quarter will get 2, the third quarter will get 3, and the last quarter will get 4.
This is equivalent to the NTILE function in SQL.
n
- (undocumented)public static Column rowNumber()
This is equivalent to the ROW_NUMBER function in SQL.
public static Column denseRank()
The difference between rank and denseRank is that denseRank leaves no gaps in ranking sequence when there are ties. That is, if you were ranking a competition using denseRank and had three people tie for second place, you would say that all three were in second place and that the next person came in third.
This is equivalent to the DENSE_RANK function in SQL.
public static Column rank()
The difference between rank and denseRank is that denseRank leaves no gaps in ranking sequence when there are ties. That is, if you were ranking a competition using denseRank and had three people tie for second place, you would say that all three were in second place and that the next person came in third.
This is equivalent to the RANK function in SQL.
public static Column cumeDist()
N = total number of rows in the partition
cumeDist(x) = number of values before (and including) x / N
This is equivalent to the CUME_DIST function in SQL.
public static Column percentRank()
This is computed by:
(rank of row in its partition - 1) / (number of rows in the partition - 1)
This is equivalent to the PERCENT_RANK function in SQL.
public static Column abs(Column e)
e
- (undocumented)public static Column array(scala.collection.Seq<Column> cols)
cols
- (undocumented)public static Column array(String colName, scala.collection.Seq<String> colNames)
colName
- (undocumented)colNames
- (undocumented)public static Column coalesce(scala.collection.Seq<Column> e)
df.select(coalesce(df("a"), df("b")))
e
- (undocumented)public static Column explode(Column e)
e
- (undocumented)public static Column lower(Column e)
e
- (undocumented)public static Column monotonicallyIncreasingId()
The generated ID is guaranteed to be monotonically increasing and unique, but not consecutive. The current implementation puts the partition ID in the upper 31 bits, and the record number within each partition in the lower 33 bits. The assumption is that the data frame has less than 1 billion partitions, and each partition has less than 8 billion records.
As an example, consider a DataFrame
with two partitions, each with 3 records.
This expression would return the following IDs:
0, 1, 2, 8589934592 (1L << 33), 8589934593, 8589934594.
public static Column negate(Column e)
// Select the amount column and negates all values.
// Scala:
df.select( -df("amount") )
// Java:
df.select( negate(df.col("amount")) );
e
- (undocumented)public static Column not(Column e)
// Scala: select rows that are not active (isActive === false)
df.filter( !df("isActive") )
// Java:
df.filter( not(df.col("isActive")) );
e
- (undocumented)public static Column when(Column condition, Object value)
// Example: encoding gender string column into integer.
// Scala:
people.select(when(people("gender") === "male", 0)
.when(people("gender") === "female", 1)
.otherwise(2))
// Java:
people.select(when(col("gender").equalTo("male"), 0)
.when(col("gender").equalTo("female"), 1)
.otherwise(2))
condition
- (undocumented)value
- (undocumented)public static Column rand(long seed)
seed
- (undocumented)public static Column rand()
public static Column randn(long seed)
seed
- (undocumented)public static Column randn()
public static Column sparkPartitionId()
Note that this is indeterministic because it depends on data partitioning and task scheduling.
public static Column sqrt(Column e)
e
- (undocumented)public static Column struct(scala.collection.Seq<Column> cols)
DataFrame
, or
a derived column expression that is named (i.e. aliased).
cols
- (undocumented)public static Column struct(String colName, scala.collection.Seq<String> colNames)
colName
- (undocumented)colNames
- (undocumented)public static Column upper(Column e)
e
- (undocumented)public static Column bitwiseNOT(Column e)
e
- (undocumented)public static Column acos(Column e)
e
- (undocumented)public static Column acos(String columnName)
columnName
- (undocumented)public static Column asin(Column e)
e
- (undocumented)public static Column asin(String columnName)
columnName
- (undocumented)public static Column atan(Column e)
e
- (undocumented)public static Column atan(String columnName)
columnName
- (undocumented)public static Column atan2(Column l, Column r)
l
- (undocumented)r
- (undocumented)public static Column atan2(Column l, String rightName)
l
- (undocumented)rightName
- (undocumented)public static Column atan2(String leftName, Column r)
leftName
- (undocumented)r
- (undocumented)public static Column atan2(String leftName, String rightName)
leftName
- (undocumented)rightName
- (undocumented)public static Column atan2(Column l, double r)
l
- (undocumented)r
- (undocumented)public static Column atan2(String leftName, double r)
leftName
- (undocumented)r
- (undocumented)public static Column atan2(double l, Column r)
l
- (undocumented)r
- (undocumented)public static Column atan2(double l, String rightName)
l
- (undocumented)rightName
- (undocumented)public static Column cbrt(Column e)
e
- (undocumented)public static Column cbrt(String columnName)
columnName
- (undocumented)public static Column ceil(Column e)
e
- (undocumented)public static Column ceil(String columnName)
columnName
- (undocumented)public static Column cos(Column e)
e
- (undocumented)public static Column cos(String columnName)
columnName
- (undocumented)public static Column cosh(Column e)
e
- (undocumented)public static Column cosh(String columnName)
columnName
- (undocumented)public static Column exp(Column e)
e
- (undocumented)public static Column exp(String columnName)
columnName
- (undocumented)public static Column expm1(Column e)
e
- (undocumented)public static Column expm1(String columnName)
columnName
- (undocumented)public static Column floor(Column e)
e
- (undocumented)public static Column floor(String columnName)
columnName
- (undocumented)public static Column hypot(Column l, Column r)
sqrt(a^2^ + b^2^)
without intermediate overflow or underflow.
l
- (undocumented)r
- (undocumented)public static Column hypot(Column l, String rightName)
sqrt(a^2^ + b^2^)
without intermediate overflow or underflow.
l
- (undocumented)rightName
- (undocumented)public static Column hypot(String leftName, Column r)
sqrt(a^2^ + b^2^)
without intermediate overflow or underflow.
leftName
- (undocumented)r
- (undocumented)public static Column hypot(String leftName, String rightName)
sqrt(a^2^ + b^2^)
without intermediate overflow or underflow.
leftName
- (undocumented)rightName
- (undocumented)public static Column hypot(Column l, double r)
sqrt(a^2^ + b^2^)
without intermediate overflow or underflow.
l
- (undocumented)r
- (undocumented)public static Column hypot(String leftName, double r)
sqrt(a^2^ + b^2^)
without intermediate overflow or underflow.
leftName
- (undocumented)r
- (undocumented)public static Column hypot(double l, Column r)
sqrt(a^2^ + b^2^)
without intermediate overflow or underflow.
l
- (undocumented)r
- (undocumented)public static Column hypot(double l, String rightName)
sqrt(a^2^ + b^2^)
without intermediate overflow or underflow.
l
- (undocumented)rightName
- (undocumented)public static Column log(Column e)
e
- (undocumented)public static Column log(String columnName)
columnName
- (undocumented)public static Column log10(Column e)
e
- (undocumented)public static Column log10(String columnName)
columnName
- (undocumented)public static Column log1p(Column e)
e
- (undocumented)public static Column log1p(String columnName)
columnName
- (undocumented)public static Column pow(Column l, Column r)
l
- (undocumented)r
- (undocumented)public static Column pow(Column l, String rightName)
l
- (undocumented)rightName
- (undocumented)public static Column pow(String leftName, Column r)
leftName
- (undocumented)r
- (undocumented)public static Column pow(String leftName, String rightName)
leftName
- (undocumented)rightName
- (undocumented)public static Column pow(Column l, double r)
l
- (undocumented)r
- (undocumented)public static Column pow(String leftName, double r)
leftName
- (undocumented)r
- (undocumented)public static Column pow(double l, Column r)
l
- (undocumented)r
- (undocumented)public static Column pow(double l, String rightName)
l
- (undocumented)rightName
- (undocumented)public static Column rint(Column e)
e
- (undocumented)public static Column rint(String columnName)
columnName
- (undocumented)public static Column signum(Column e)
e
- (undocumented)public static Column signum(String columnName)
columnName
- (undocumented)public static Column sin(Column e)
e
- (undocumented)public static Column sin(String columnName)
columnName
- (undocumented)public static Column sinh(Column e)
e
- (undocumented)public static Column sinh(String columnName)
columnName
- (undocumented)public static Column tan(Column e)
e
- (undocumented)public static Column tan(String columnName)
columnName
- (undocumented)public static Column tanh(Column e)
e
- (undocumented)public static Column tanh(String columnName)
columnName
- (undocumented)public static Column toDegrees(Column e)
e
- (undocumented)public static Column toDegrees(String columnName)
columnName
- (undocumented)public static Column toRadians(Column e)
e
- (undocumented)public static Column toRadians(String columnName)
columnName
- (undocumented)public static <RT> UserDefinedFunction udf(scala.Function0<RT> f, scala.reflect.api.TypeTags.TypeTag<RT> evidence$1)
f
- (undocumented)evidence$1
- (undocumented)public static <RT,A1> UserDefinedFunction udf(scala.Function1<A1,RT> f, scala.reflect.api.TypeTags.TypeTag<RT> evidence$2, scala.reflect.api.TypeTags.TypeTag<A1> evidence$3)
f
- (undocumented)evidence$2
- (undocumented)evidence$3
- (undocumented)public static <RT,A1,A2> UserDefinedFunction udf(scala.Function2<A1,A2,RT> f, scala.reflect.api.TypeTags.TypeTag<RT> evidence$4, scala.reflect.api.TypeTags.TypeTag<A1> evidence$5, scala.reflect.api.TypeTags.TypeTag<A2> evidence$6)
f
- (undocumented)evidence$4
- (undocumented)evidence$5
- (undocumented)evidence$6
- (undocumented)public static <RT,A1,A2,A3> UserDefinedFunction udf(scala.Function3<A1,A2,A3,RT> f, scala.reflect.api.TypeTags.TypeTag<RT> evidence$7, scala.reflect.api.TypeTags.TypeTag<A1> evidence$8, scala.reflect.api.TypeTags.TypeTag<A2> evidence$9, scala.reflect.api.TypeTags.TypeTag<A3> evidence$10)
f
- (undocumented)evidence$7
- (undocumented)evidence$8
- (undocumented)evidence$9
- (undocumented)evidence$10
- (undocumented)public static <RT,A1,A2,A3,A4> UserDefinedFunction udf(scala.Function4<A1,A2,A3,A4,RT> f, scala.reflect.api.TypeTags.TypeTag<RT> evidence$11, scala.reflect.api.TypeTags.TypeTag<A1> evidence$12, scala.reflect.api.TypeTags.TypeTag<A2> evidence$13, scala.reflect.api.TypeTags.TypeTag<A3> evidence$14, scala.reflect.api.TypeTags.TypeTag<A4> evidence$15)
f
- (undocumented)evidence$11
- (undocumented)evidence$12
- (undocumented)evidence$13
- (undocumented)evidence$14
- (undocumented)evidence$15
- (undocumented)public static <RT,A1,A2,A3,A4,A5> UserDefinedFunction udf(scala.Function5<A1,A2,A3,A4,A5,RT> f, scala.reflect.api.TypeTags.TypeTag<RT> evidence$16, scala.reflect.api.TypeTags.TypeTag<A1> evidence$17, scala.reflect.api.TypeTags.TypeTag<A2> evidence$18, scala.reflect.api.TypeTags.TypeTag<A3> evidence$19, scala.reflect.api.TypeTags.TypeTag<A4> evidence$20, scala.reflect.api.TypeTags.TypeTag<A5> evidence$21)
f
- (undocumented)evidence$16
- (undocumented)evidence$17
- (undocumented)evidence$18
- (undocumented)evidence$19
- (undocumented)evidence$20
- (undocumented)evidence$21
- (undocumented)public static <RT,A1,A2,A3,A4,A5,A6> UserDefinedFunction udf(scala.Function6<A1,A2,A3,A4,A5,A6,RT> f, scala.reflect.api.TypeTags.TypeTag<RT> evidence$22, scala.reflect.api.TypeTags.TypeTag<A1> evidence$23, scala.reflect.api.TypeTags.TypeTag<A2> evidence$24, scala.reflect.api.TypeTags.TypeTag<A3> evidence$25, scala.reflect.api.TypeTags.TypeTag<A4> evidence$26, scala.reflect.api.TypeTags.TypeTag<A5> evidence$27, scala.reflect.api.TypeTags.TypeTag<A6> evidence$28)
f
- (undocumented)evidence$22
- (undocumented)evidence$23
- (undocumented)evidence$24
- (undocumented)evidence$25
- (undocumented)evidence$26
- (undocumented)evidence$27
- (undocumented)evidence$28
- (undocumented)public static <RT,A1,A2,A3,A4,A5,A6,A7> UserDefinedFunction udf(scala.Function7<A1,A2,A3,A4,A5,A6,A7,RT> f, scala.reflect.api.TypeTags.TypeTag<RT> evidence$29, scala.reflect.api.TypeTags.TypeTag<A1> evidence$30, scala.reflect.api.TypeTags.TypeTag<A2> evidence$31, scala.reflect.api.TypeTags.TypeTag<A3> evidence$32, scala.reflect.api.TypeTags.TypeTag<A4> evidence$33, scala.reflect.api.TypeTags.TypeTag<A5> evidence$34, scala.reflect.api.TypeTags.TypeTag<A6> evidence$35, scala.reflect.api.TypeTags.TypeTag<A7> evidence$36)
f
- (undocumented)evidence$29
- (undocumented)evidence$30
- (undocumented)evidence$31
- (undocumented)evidence$32
- (undocumented)evidence$33
- (undocumented)evidence$34
- (undocumented)evidence$35
- (undocumented)evidence$36
- (undocumented)public static <RT,A1,A2,A3,A4,A5,A6,A7,A8> UserDefinedFunction udf(scala.Function8<A1,A2,A3,A4,A5,A6,A7,A8,RT> f, scala.reflect.api.TypeTags.TypeTag<RT> evidence$37, scala.reflect.api.TypeTags.TypeTag<A1> evidence$38, scala.reflect.api.TypeTags.TypeTag<A2> evidence$39, scala.reflect.api.TypeTags.TypeTag<A3> evidence$40, scala.reflect.api.TypeTags.TypeTag<A4> evidence$41, scala.reflect.api.TypeTags.TypeTag<A5> evidence$42, scala.reflect.api.TypeTags.TypeTag<A6> evidence$43, scala.reflect.api.TypeTags.TypeTag<A7> evidence$44, scala.reflect.api.TypeTags.TypeTag<A8> evidence$45)
f
- (undocumented)evidence$37
- (undocumented)evidence$38
- (undocumented)evidence$39
- (undocumented)evidence$40
- (undocumented)evidence$41
- (undocumented)evidence$42
- (undocumented)evidence$43
- (undocumented)evidence$44
- (undocumented)evidence$45
- (undocumented)public static <RT,A1,A2,A3,A4,A5,A6,A7,A8,A9> UserDefinedFunction udf(scala.Function9<A1,A2,A3,A4,A5,A6,A7,A8,A9,RT> f, scala.reflect.api.TypeTags.TypeTag<RT> evidence$46, scala.reflect.api.TypeTags.TypeTag<A1> evidence$47, scala.reflect.api.TypeTags.TypeTag<A2> evidence$48, scala.reflect.api.TypeTags.TypeTag<A3> evidence$49, scala.reflect.api.TypeTags.TypeTag<A4> evidence$50, scala.reflect.api.TypeTags.TypeTag<A5> evidence$51, scala.reflect.api.TypeTags.TypeTag<A6> evidence$52, scala.reflect.api.TypeTags.TypeTag<A7> evidence$53, scala.reflect.api.TypeTags.TypeTag<A8> evidence$54, scala.reflect.api.TypeTags.TypeTag<A9> evidence$55)
f
- (undocumented)evidence$46
- (undocumented)evidence$47
- (undocumented)evidence$48
- (undocumented)evidence$49
- (undocumented)evidence$50
- (undocumented)evidence$51
- (undocumented)evidence$52
- (undocumented)evidence$53
- (undocumented)evidence$54
- (undocumented)evidence$55
- (undocumented)public static <RT,A1,A2,A3,A4,A5,A6,A7,A8,A9,A10> UserDefinedFunction udf(scala.Function10<A1,A2,A3,A4,A5,A6,A7,A8,A9,A10,RT> f, scala.reflect.api.TypeTags.TypeTag<RT> evidence$56, scala.reflect.api.TypeTags.TypeTag<A1> evidence$57, scala.reflect.api.TypeTags.TypeTag<A2> evidence$58, scala.reflect.api.TypeTags.TypeTag<A3> evidence$59, scala.reflect.api.TypeTags.TypeTag<A4> evidence$60, scala.reflect.api.TypeTags.TypeTag<A5> evidence$61, scala.reflect.api.TypeTags.TypeTag<A6> evidence$62, scala.reflect.api.TypeTags.TypeTag<A7> evidence$63, scala.reflect.api.TypeTags.TypeTag<A8> evidence$64, scala.reflect.api.TypeTags.TypeTag<A9> evidence$65, scala.reflect.api.TypeTags.TypeTag<A10> evidence$66)
f
- (undocumented)evidence$56
- (undocumented)evidence$57
- (undocumented)evidence$58
- (undocumented)evidence$59
- (undocumented)evidence$60
- (undocumented)evidence$61
- (undocumented)evidence$62
- (undocumented)evidence$63
- (undocumented)evidence$64
- (undocumented)evidence$65
- (undocumented)evidence$66
- (undocumented)public static Column callUDF(scala.Function0<?> f, DataType returnType)
f
- (undocumented)returnType
- (undocumented)public static Column callUDF(scala.Function1<?,?> f, DataType returnType, Column arg1)
f
- (undocumented)returnType
- (undocumented)arg1
- (undocumented)public static Column callUDF(scala.Function2<?,?,?> f, DataType returnType, Column arg1, Column arg2)
f
- (undocumented)returnType
- (undocumented)arg1
- (undocumented)arg2
- (undocumented)public static Column callUDF(scala.Function3<?,?,?,?> f, DataType returnType, Column arg1, Column arg2, Column arg3)
f
- (undocumented)returnType
- (undocumented)arg1
- (undocumented)arg2
- (undocumented)arg3
- (undocumented)public static Column callUDF(scala.Function4<?,?,?,?,?> f, DataType returnType, Column arg1, Column arg2, Column arg3, Column arg4)
f
- (undocumented)returnType
- (undocumented)arg1
- (undocumented)arg2
- (undocumented)arg3
- (undocumented)arg4
- (undocumented)public static Column callUDF(scala.Function5<?,?,?,?,?,?> f, DataType returnType, Column arg1, Column arg2, Column arg3, Column arg4, Column arg5)
f
- (undocumented)returnType
- (undocumented)arg1
- (undocumented)arg2
- (undocumented)arg3
- (undocumented)arg4
- (undocumented)arg5
- (undocumented)public static Column callUDF(scala.Function6<?,?,?,?,?,?,?> f, DataType returnType, Column arg1, Column arg2, Column arg3, Column arg4, Column arg5, Column arg6)
f
- (undocumented)returnType
- (undocumented)arg1
- (undocumented)arg2
- (undocumented)arg3
- (undocumented)arg4
- (undocumented)arg5
- (undocumented)arg6
- (undocumented)public static Column callUDF(scala.Function7<?,?,?,?,?,?,?,?> f, DataType returnType, Column arg1, Column arg2, Column arg3, Column arg4, Column arg5, Column arg6, Column arg7)
f
- (undocumented)returnType
- (undocumented)arg1
- (undocumented)arg2
- (undocumented)arg3
- (undocumented)arg4
- (undocumented)arg5
- (undocumented)arg6
- (undocumented)arg7
- (undocumented)public static Column callUDF(scala.Function8<?,?,?,?,?,?,?,?,?> f, DataType returnType, Column arg1, Column arg2, Column arg3, Column arg4, Column arg5, Column arg6, Column arg7, Column arg8)
f
- (undocumented)returnType
- (undocumented)arg1
- (undocumented)arg2
- (undocumented)arg3
- (undocumented)arg4
- (undocumented)arg5
- (undocumented)arg6
- (undocumented)arg7
- (undocumented)arg8
- (undocumented)public static Column callUDF(scala.Function9<?,?,?,?,?,?,?,?,?,?> f, DataType returnType, Column arg1, Column arg2, Column arg3, Column arg4, Column arg5, Column arg6, Column arg7, Column arg8, Column arg9)
f
- (undocumented)returnType
- (undocumented)arg1
- (undocumented)arg2
- (undocumented)arg3
- (undocumented)arg4
- (undocumented)arg5
- (undocumented)arg6
- (undocumented)arg7
- (undocumented)arg8
- (undocumented)arg9
- (undocumented)public static Column callUDF(scala.Function10<?,?,?,?,?,?,?,?,?,?,?> f, DataType returnType, Column arg1, Column arg2, Column arg3, Column arg4, Column arg5, Column arg6, Column arg7, Column arg8, Column arg9, Column arg10)
f
- (undocumented)returnType
- (undocumented)arg1
- (undocumented)arg2
- (undocumented)arg3
- (undocumented)arg4
- (undocumented)arg5
- (undocumented)arg6
- (undocumented)arg7
- (undocumented)arg8
- (undocumented)arg9
- (undocumented)arg10
- (undocumented)public static Column callUdf(String udfName, scala.collection.Seq<Column> cols)
import org.apache.spark.sql._
val df = Seq(("id1", 1), ("id2", 4), ("id3", 5)).toDF("id", "value")
val sqlContext = df.sqlContext
sqlContext.udf.register("simpleUdf", (v: Int) => v * v)
df.select($"id", callUdf("simpleUdf", $"value"))
udfName
- (undocumented)cols
- (undocumented)