Column

Value Members

1. final def !=(arg0: AnyRef): Boolean

   Definition Classes

   AnyRef

2. final def !=(arg0: Any): Boolean

   Definition Classes

   Any

3. def !==(other: Any): Column

   Inequality test.

   Inequality test.

   // Scala:
   df.select( df("colA") !== df("colB") )
   df.select( !(df("colA") === df("colB")) )
   
   // Java:
   import static org.apache.spark.sql.functions.*;
   df.filter( col("colA").notEqual(col("colB")) );

4. final def ##(): Int

   Definition Classes

   AnyRef → Any

5. def %(other: Any): Column

   Modulo (a.

   Modulo (a.k.a. remainder) expression.

6. def &&(other: Any): Column

   Boolean AND.

   Boolean AND.

   // Scala: The following selects people that are in school and employed at the same time.
   people.select( people("inSchool") && people("isEmployed") )
   
   // Java:
   people.select( people("inSchool").and(people("isEmployed")) );

7. def *(other: Any): Column

   Multiplication of this expression and another expression.

   Multiplication of this expression and another expression.

   // Scala: The following multiplies a person's height by their weight.
   people.select( people("height") * people("weight") )
   
   // Java:
   people.select( people("height").multiply(people("weight")) );

8. def +(other: Any): Column

   Sum of this expression and another expression.

   Sum of this expression and another expression.

   // Scala: The following selects the sum of a person's height and weight.
   people.select( people("height") + people("weight") )
   
   // Java:
   people.select( people("height").plus(people("weight")) );

9. def -(other: Any): Column

   Subtraction.

   Subtraction. Subtract the other expression from this expression.

   // Scala: The following selects the difference between people's height and their weight.
   people.select( people("height") - people("weight") )
   
   // Java:
   people.select( people("height").minus(people("weight")) );

10. def /(other: Any): Column

    Division this expression by another expression.

    Division this expression by another expression.

    // Scala: The following divides a person's height by their weight.
    people.select( people("height") / people("weight") )
    
    // Java:
    people.select( people("height").divide(people("weight")) );

11. def <(other: Any): Column

    Less than.

    Less than.

    // Scala: The following selects people younger than 21.
    people.select( people("age") < 21 )
    
    // Java:
    people.select( people("age").lt(21) );

12. def <=(other: Any): Column

    Less than or equal to.

    Less than or equal to.

    // Scala: The following selects people age 21 or younger than 21.
    people.select( people("age") <= 21 )
    
    // Java:
    people.select( people("age").leq(21) );

13. def <=>(other: Any): Column

    Equality test that is safe for null values.

14. final def ==(arg0: AnyRef): Boolean

    Definition Classes

    AnyRef

15. final def ==(arg0: Any): Boolean

    Definition Classes

    Any

16. def ===(other: Any): Column

    Equality test.

    Equality test.

    // Scala:
    df.filter( df("colA") === df("colB") )
    
    // Java
    import static org.apache.spark.sql.functions.*;
    df.filter( col("colA").equalTo(col("colB")) );

17. def >(other: Any): Column

    Greater than.

    Greater than.

    // Scala: The following selects people older than 21.
    people.select( people("age") > 21 )
    
    // Java:
    import static org.apache.spark.sql.functions.*;
    people.select( people("age").gt(21) );

18. def >=(other: Any): Column

    Greater than or equal to an expression.

    Greater than or equal to an expression.

    // Scala: The following selects people age 21 or older than 21.
    people.select( people("age") >= 21 )
    
    // Java:
    people.select( people("age").geq(21) )

19. def and(other: Column): Column

    Boolean AND.

    Boolean AND.

    // Scala: The following selects people that are in school and employed at the same time.
    people.select( people("inSchool") && people("isEmployed") )
    
    // Java:
    people.select( people("inSchool").and(people("isEmployed")) );

20. def as(alias: Symbol): Column

    Gives the column an alias.

    Gives the column an alias.

    // Renames colA to colB in select output.
    df.select($"colA".as('colB))

21. def as(alias: String): Column

    Gives the column an alias.

    Gives the column an alias.

    // Renames colA to colB in select output.
    df.select($"colA".as("colB"))

22. final def asInstanceOf[T0]: T0

    Definition Classes

    Any

23. def asc: Column

    Returns an ordering used in sorting.

    Returns an ordering used in sorting.

    // Scala: sort a DataFrame by age column in ascending order.
    df.sort(df("age").asc)
    
    // Java
    df.sort(df.col("age").asc());

24. def cast(to: String): Column

    Casts the column to a different data type, using the canonical string representation of the type.

    Casts the column to a different data type, using the canonical string representation of the type. The supported types are: string, boolean, byte, short, int, long, float, double, decimal, date, timestamp.

    // Casts colA to integer.
    df.select(df("colA").cast("int"))

25. def cast(to: DataType): Column

    Casts the column to a different data type.

    Casts the column to a different data type.

    // Casts colA to IntegerType.
    import org.apache.spark.sql.types.IntegerType
    df.select(df("colA").cast(IntegerType))
    
    // equivalent to
    df.select(df("colA").cast("int"))

26. def clone(): AnyRef

    Attributes

    protected[java.lang]

    Definition Classes

    AnyRef

    Annotations

    @throws( ... )

27. def contains(other: Any): Column

    Contains the other element.

28. def desc: Column

    Returns an ordering used in sorting.

    Returns an ordering used in sorting.

    // Scala: sort a DataFrame by age column in descending order.
    df.sort(df("age").desc)
    
    // Java
    df.sort(df.col("age").desc());

29. def divide(other: Any): Column

    Division this expression by another expression.

    Division this expression by another expression.

    // Scala: The following divides a person's height by their weight.
    people.select( people("height") / people("weight") )
    
    // Java:
    people.select( people("height").divide(people("weight")) );

30. def endsWith(literal: String): Column

    String ends with another string literal.

31. def endsWith(other: Column): Column

    String ends with.

32. final def eq(arg0: AnyRef): Boolean

    Definition Classes

    AnyRef

33. def eqNullSafe(other: Any): Column

    Equality test that is safe for null values.

34. def equalTo(other: Any): Column

    Equality test.

    Equality test.

    // Scala:
    df.filter( df("colA") === df("colB") )
    
    // Java
    import static org.apache.spark.sql.functions.*;
    df.filter( col("colA").equalTo(col("colB")) );

35. def equals(that: Any): Boolean

    Definition Classes

    Column → AnyRef → Any

36. def explain(extended: Boolean): Unit

    Prints the expression to the console for debugging purpose.

37. val expr: Expression

    Attributes

    protected[org.apache.spark.sql]

38. def finalize(): Unit

    Attributes

    protected[java.lang]

    Definition Classes

    AnyRef

    Annotations

    @throws( classOf[java.lang.Throwable] )

39. def geq(other: Any): Column

    Greater than or equal to an expression.

    Greater than or equal to an expression.

    // Scala: The following selects people age 21 or older than 21.
    people.select( people("age") >= 21 )
    
    // Java:
    people.select( people("age").geq(21) )

40. final def getClass(): Class[_]

    Definition Classes

    AnyRef → Any

41. def getField(fieldName: String): Column

    An expression that gets a field by name in a StructField.

42. def getItem(ordinal: Int): Column

    An expression that gets an item at position ordinal out of an array.

43. def gt(other: Any): Column

    Greater than.

    Greater than.

    // Scala: The following selects people older than 21.
    people.select( people("age") > lit(21) )
    
    // Java:
    import static org.apache.spark.sql.functions.*;
    people.select( people("age").gt(21) );

44. def hashCode(): Int

    Definition Classes

    Column → AnyRef → Any

45. def in(list: Column*): Column

    A boolean expression that is evaluated to true if the value of this expression is contained by the evaluated values of the arguments.

    A boolean expression that is evaluated to true if the value of this expression is contained by the evaluated values of the arguments.

    Annotations

    @varargs()

46. final def isInstanceOf[T0]: Boolean

    Definition Classes

    Any

47. def isNotNull: Column

    True if the current expression is NOT null.

48. def isNull: Column

    True if the current expression is null.

49. def isTraceEnabled(): Boolean

    Attributes

    protected

    Definition Classes

    Logging

50. def leq(other: Any): Column

    Less than or equal to.

    Less than or equal to.

    // Scala: The following selects people age 21 or younger than 21.
    people.select( people("age") <= 21 )
    
    // Java:
    people.select( people("age").leq(21) );

51. def like(literal: String): Column

    SQL like expression.

52. def log: Logger

    Attributes

    protected

    Definition Classes

    Logging

53. def logDebug(msg: ⇒ String, throwable: Throwable): Unit

    Attributes

    protected

    Definition Classes

    Logging

54. def logDebug(msg: ⇒ String): Unit

    Attributes

    protected

    Definition Classes

    Logging

55. def logError(msg: ⇒ String, throwable: Throwable): Unit

    Attributes

    protected

    Definition Classes

    Logging

56. def logError(msg: ⇒ String): Unit

    Attributes

    protected

    Definition Classes

    Logging

57. def logInfo(msg: ⇒ String, throwable: Throwable): Unit

    Attributes

    protected

    Definition Classes

    Logging

58. def logInfo(msg: ⇒ String): Unit

    Attributes

    protected

    Definition Classes

    Logging

59. def logName: String

    Attributes

    protected

    Definition Classes

    Logging

60. def logTrace(msg: ⇒ String, throwable: Throwable): Unit

    Attributes

    protected

    Definition Classes

    Logging

61. def logTrace(msg: ⇒ String): Unit

    Attributes

    protected

    Definition Classes

    Logging

62. def logWarning(msg: ⇒ String, throwable: Throwable): Unit

    Attributes

    protected

    Definition Classes

    Logging

63. def logWarning(msg: ⇒ String): Unit

    Attributes

    protected

    Definition Classes

    Logging

64. def lt(other: Any): Column

    Less than.

    Less than.

    // Scala: The following selects people younger than 21.
    people.select( people("age") < 21 )
    
    // Java:
    people.select( people("age").lt(21) );

65. def minus(other: Any): Column

    Subtraction.

    Subtraction. Subtract the other expression from this expression.

    // Scala: The following selects the difference between people's height and their weight.
    people.select( people("height") - people("weight") )
    
    // Java:
    people.select( people("height").minus(people("weight")) );

66. def mod(other: Any): Column

    Modulo (a.

    Modulo (a.k.a. remainder) expression.

67. def multiply(other: Any): Column

    Multiplication of this expression and another expression.

    Multiplication of this expression and another expression.

    // Scala: The following multiplies a person's height by their weight.
    people.select( people("height") * people("weight") )
    
    // Java:
    people.select( people("height").multiply(people("weight")) );

68. final def ne(arg0: AnyRef): Boolean

    Definition Classes

    AnyRef

69. def notEqual(other: Any): Column

    Inequality test.

    Inequality test.

    // Scala:
    df.select( df("colA") !== df("colB") )
    df.select( !(df("colA") === df("colB")) )
    
    // Java:
    import static org.apache.spark.sql.functions.*;
    df.filter( col("colA").notEqual(col("colB")) );

70. final def notify(): Unit

    Definition Classes

    AnyRef

71. final def notifyAll(): Unit

    Definition Classes

    AnyRef

72. def or(other: Column): Column

    Boolean OR.

    Boolean OR.

    // Scala: The following selects people that are in school or employed.
    people.filter( people("inSchool") || people("isEmployed") )
    
    // Java:
    people.filter( people("inSchool").or(people("isEmployed")) );

73. def plus(other: Any): Column

    Sum of this expression and another expression.

    Sum of this expression and another expression.

    // Scala: The following selects the sum of a person's height and weight.
    people.select( people("height") + people("weight") )
    
    // Java:
    people.select( people("height").plus(people("weight")) );

74. def rlike(literal: String): Column

    SQL RLIKE expression (LIKE with Regex).

75. def startsWith(literal: String): Column

    String starts with another string literal.

76. def startsWith(other: Column): Column

    String starts with.

77. def substr(startPos: Int, len: Int): Column

    An expression that returns a substring.

    An expression that returns a substring.

    startPos

    starting position.

    len

    length of the substring.

78. def substr(startPos: Column, len: Column): Column

    An expression that returns a substring.

    An expression that returns a substring.

    startPos

    expression for the starting position.

    len

    expression for the length of the substring.

79. final def synchronized[T0](arg0: ⇒ T0): T0

    Definition Classes

    AnyRef

80. def toString(): String

    Definition Classes

    Column → AnyRef → Any

81. def unary_!: Column

    Inversion of boolean expression, i.

    Inversion of boolean expression, i.e. NOT. {{ // Scala: select rows that are not active (isActive === false) df.filter( !df("isActive") )

    // Java: import static org.apache.spark.sql.functions.*; df.filter( not(df.col("isActive")) ); }}

82. def unary_-: Column

    Unary minus, i.

    Unary minus, i.e. negate the expression.

    // Scala: select the amount column and negates all values.
    df.select( -df("amount") )
    
    // Java:
    import static org.apache.spark.sql.functions.*;
    df.select( negate(col("amount") );

83. final def wait(): Unit

    Definition Classes

    AnyRef

    Annotations

    @throws( ... )

84. final def wait(arg0: Long, arg1: Int): Unit

    Definition Classes

    AnyRef

    Annotations

    @throws( ... )

85. final def wait(arg0: Long): Unit

    Definition Classes

    AnyRef

    Annotations

    @throws( ... )

86. def ||(other: Any): Column

    Boolean OR.

    Boolean OR.

    // Scala: The following selects people that are in school or employed.
    people.filter( people("inSchool") || people("isEmployed") )
    
    // Java:
    people.filter( people("inSchool").or(people("isEmployed")) );