Great Expectations

Set

Pandas

import great_expectations as ge

df = ge.read_csv('https://raw.githubusercontent.com/tidyverse/ggplot2/master/data-raw/diamonds.csv')
df.expect_column_values_to_not_be_null('cut')
df.expect_column_values_to_be_between('carat', min_value=0, max_value=4)
df.save_expectation_suite('../../data/diamonds_csv_expectations.json')

Integrate with DAG Tools

Prefect

from prefect import task, Flow, Parameter
from prefect.tasks.great_expectations import RunGreatExpectationsValidation

def make_params(names:list):
    return {name: Parameter(name) for name in names}

with Flow('test') as flow:
    params = make_params(['dataset', 'expectation_suite_name'])
    
    df = ge.read_csv(parmas['dataset'])