33 lines
833 B
YAML
33 lines
833 B
YAML
name: Nightly dry-run
|
|
|
|
on:
|
|
schedule:
|
|
- cron: '0 2 * * *'
|
|
|
|
jobs:
|
|
dry-run:
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
|
|
- name: Set up Python
|
|
uses: actions/setup-python@v5
|
|
with:
|
|
python-version: '3.11'
|
|
|
|
- name: Install dependencies
|
|
run: pip install -r requirements.txt
|
|
|
|
- name: Dry-run pipeline
|
|
run: |
|
|
python3 -c "
|
|
import sys
|
|
sys.path.insert(0, 'src')
|
|
from transform.clean import drop_nulls, deduplicate
|
|
import pandas as pd
|
|
df = pd.DataFrame({'id': [1, 1, 2, None], 'val': ['a', 'a', 'b', 'c']})
|
|
df = drop_nulls(df, ['id'])
|
|
df = deduplicate(df, ['id'])
|
|
assert len(df) == 2, f'Expected 2 rows, got {len(df)}'
|
|
print('Dry-run OK')
|
|
"
|