Package com.dataai.etl.spark.functions

Class DataQualityFunctions

java.lang.Object
com.dataai.etl.spark.functions.DataQualityFunctions
public final class DataQualityFunctions extends Object

  • Method Summary

    Modifier and Type
    Method
    Description
    static org.apache.spark.sql.Dataset<org.apache.spark.sql.Row>
    automaticChecks(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> input, double standardDeviationThreshold)
     
    static org.apache.spark.sql.Dataset<org.apache.spark.sql.Row>
    duplicateRecordSummary(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> input)
     
    static org.apache.spark.sql.Dataset<org.apache.spark.sql.Row>
    inconsistentCategorySummary(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> input)
     
    static org.apache.spark.sql.Dataset<org.apache.spark.sql.Row>
    invalidDateSummary(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> input)
     
    static org.apache.spark.sql.Dataset<org.apache.spark.sql.Row>
    missingValueSummary(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> input)
     
    static org.apache.spark.sql.Dataset<org.apache.spark.sql.Row>
    numericOutlierSummary(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> input, double threshold)
     
    static org.apache.spark.sql.Dataset<org.apache.spark.sql.Row>
    suspiciousTextSummary(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> input)
     

    Methods inherited from class java.lang.Object

    clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

  • Method Details

    • automaticChecks

      public static org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> automaticChecks(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> input, double standardDeviationThreshold)

    • missingValueSummary

      public static org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> missingValueSummary(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> input)

    • duplicateRecordSummary

      public static org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> duplicateRecordSummary(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> input)

    • invalidDateSummary

      public static org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> invalidDateSummary(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> input)

    • numericOutlierSummary

      public static org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> numericOutlierSummary(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> input, double threshold)

    • inconsistentCategorySummary

      public static org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> inconsistentCategorySummary(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> input)

    • suspiciousTextSummary

      public static org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> suspiciousTextSummary(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> input)