Multi-files API Processing

pip install "unstructured-ingest[s3]"

from unstructured_ingest.interfaces import (
    FsspecConfig,
    PartitionConfig,
    ProcessorConfig,
    ReadConfig,
)
from unstructured_ingest.runner import S3Runner

from unstructured.chunking.title import chunk_by_title
from unstructured.staging.base import dict_to_elements

UNSTRUCTURED_API_KEY = os.getenv('UNSTRUCTURED_API_KEY')
S3_URL = "s3://rh-financial-reports/world-development-bank-2023/"

runner = S3Runner(
     processor_config=ProcessorConfig(
         verbose=True,
         output_dir=os.getenv("LOCAL_FILE_OUTPUT_DIR"),
         num_processes=8,
     ),
     read_config=ReadConfig(),
     partition_config=PartitionConfig(
         partition_endpoint=os.getenv("UNSTRUCTURED_API_URL"),
         partition_by_api=True,
         api_key=os.getenv("UNSTRUCTURED_API_KEY"),
         strategy="hi_res",
         hi_res_model_name="yolox",
     ),
     fsspec_config=FsspecConfig(
         remote_url=S3_URL,
     ),
 )

runner.run(anonymous=True)

combined_json_data = read_and_combine_json("Connector-Output/world-development-bank-2023")

elements = dict_to_elements(combined_json_data)
chunks = chunk_by_title(elements)