Delta Table

#!/usr/bin/env bash

unstructured-ingest \
  delta-table \
    --table-uri s3://utic-dev-tech-fixtures/sample-delta-lake-data/deltatable/ \
    --output-dir $LOCAL_FILE_OUTPUT_DIR \
    --storage_options "AWS_REGION=us-east-2,AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID,AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY" \
    --verbose \
    --strategy hi_res

import os

from unstructured_ingest.connector.delta_table import SimpleDeltaTableConfig
from unstructured_ingest.interfaces import PartitionConfig, ProcessorConfig, ReadConfig
from unstructured_ingest.runner import DeltaTableRunner

if __name__ == "__main__":
    runner = DeltaTableRunner(
        processor_config=ProcessorConfig(
            verbose=True,
            output_dir=os.getenv("LOCAL_FILE_OUTPUT_DIR"),
            num_processes=2,
        ),
        read_config=ReadConfig(),
        partition_config=PartitionConfig(
            strategy="hi_res",
        ),
        connector_config=SimpleDeltaTableConfig(
            table_uri="s3://utic-dev-tech-fixtures/sample-delta-lake-data/deltatable/",
            storage_options={
                "AWS_REGION": "us-east-2",
                "AWS_ACCESS_KEY_ID": os.getenv("AWS_ACCESS_KEY_ID"),
                "AWS_SECRET_ACCESS_KEY": os.getenv("AWS_SECRET_ACCESS_KEY"),
            },
        ),
    )
    runner.run()