Package com.dataai.etl.spark.core

Class DataAiPipeline

java.lang.Object

com.dataai.etl.spark.core.DataAiPipeline

public final class DataAiPipeline
extends Object

Field Summary

Fields

Modifier and Type	Field	Description
static final String	LIBRARY_VERSION

Method Summary

Modifier and Type	Method	Description
com.dataai.etl.spark.api.DataAiResult	execute()
static DataAiPipeline	fromDataset(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> source)
static DataAiPipeline	fromTable(org.apache.spark.sql.SparkSession spark, String table)
DataAiPipeline	normalize()
DataAiPipeline	profile()
DataAiPipeline	recordKey(String... fields)
DataAiPipeline	validate(com.dataai.etl.spark.api.RuleSpec... rules)
DataAiPipeline	validate(List<com.dataai.etl.spark.api.RuleSpec> rules)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Details

LIBRARY_VERSION

public static final String LIBRARY_VERSION

See Also:

• Constant Field Values

Method Details

fromTable

public static DataAiPipeline fromTable(org.apache.spark.sql.SparkSession spark,
 String table)

fromDataset

public static DataAiPipeline fromDataset(org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> source)

normalize

public DataAiPipeline normalize()

profile

public DataAiPipeline profile()

recordKey

public DataAiPipeline recordKey(String... fields)

validate

public DataAiPipeline validate(com.dataai.etl.spark.api.RuleSpec... rules)

validate

public DataAiPipeline validate(List<com.dataai.etl.spark.api.RuleSpec> rules)

execute

public com.dataai.etl.spark.api.DataAiResult execute()