import pandas as pd
from azure.data.factory import *
def build_pipeline(source, sink):
df = pd.read_sql(query, conn)
df = df.dropna(subset=['id'])
df['quality_score'] = validate(df)
return transform(df)
class DataGovernance:
def __init__(self, config):
self.rules = load_rules(config)
def profile(self, dataset):
return {col: stats(dataset[col]) for col in dataset}
SELECT d.*, q.score
FROM datasets d
JOIN quality_checks q ON d.id = q.dataset_id
WHERE q.score > 0.95
ORDER BY d.created_at DESC;
pipeline = DataPipeline(
extract=SQLSource(conn),
transform=[Clean(), Enrich(), Validate()],
load=AzureSink(container)
)
pipeline.run(schedule='daily')
// Hello World, I'm
Roham Chlak
رُهام شلاق
Data professional with hands-on experience in data engineering, data quality, governance, and analytics. Skilled in building scalable ETL pipelines, profiling datasets, and delivering actionable insights across enterprise organizations in the UAE. Passionate about turning messy data into clean, reliable, governed assets.