Need help with complex pdf with tables across multiple pages

[learning-frog]

I am trying to extract table data from this PDF, which contains two types of tables that span multiple pages:

Tables with the same column names spread across multiple pages (pages 62-64).
Tables with different columns across multiple pages (pages 36-37).
I’m using the following logic to join tables that span multiple pages:

I maintain two counters: one for the previous table and one for the current table.
If the previous table is on the right, the current table is on top, and the headers don’t match, I classify it as the second type of table.
If the previous table is on the right, the current table is on top, and the headers match, I classify it as the first type of table.
However, I’m struggling to extract these tables using the correct table settings. I’ve experimented with different configurations but haven’t been able to extract them successfully.

Any advice or assistance would be greatly appreciated!

import pdfplumber
import pandas as pd
from pdfplumber.table import TableSettings


def is_current_table_at_top(page_height, table_bbox, top_threshold=0.1):
    top_of_table = table_bbox[1]
    return top_of_table <= top_threshold * page_height

def is_prev_table_at_right(page_width, table_bbox, right_threshold=0.1):
    right_of_table = table_bbox[2]
    right_margin = page_width - right_of_table
    #Check if table margin is less tan 1 % or even less than page width meaning it's overlapping
    return right_of_table >= (1 - right_threshold) * page_width and right_margin < right_threshold * page_width


def merge_tables_by_index(t1, t2):
    df1 = pd.DataFrame(t1[1:], columns=t1[0])
    df2 = pd.DataFrame(t2[1:], columns=t2[0])

    # Concatenate along the columns and remove columns that are entirely empty
    merged_df = pd.concat([df1, df2], axis=1)
    merged_df = merged_df.dropna(axis=1, how='all')  # Drop columns where all values are NaN

    return merged_df

def merge_tables_by_columns(t1, t2):
    df1 = pd.DataFrame(t1[1:], columns=t1[0])
    df2 = pd.DataFrame(t2[1:], columns=t2[0])

    # Concatenate along the columns and remove columns that are entirely empty
    merged_df = pd.concat([df1, df2], ignore_index=True)
    merged_df = merged_df.dropna(axis=1, how='all')  # Drop columns where all values are NaN

    return merged_df

def extract_table_header(table_data):
    if table_data:
        header = [col for col in table_data[0] if col is not None]  # Remove None values
        return header
    return None

def extract_tables_with_position_and_merge(pdf_path):
    previous_table = None
    merged_table_count = 0

    with pdfplumber.open(pdf_path) as pdf:
        for i, page in enumerate(pdf.pages):
            table_settings = {
                "vertical_strategy": "lines",
                "horizontal_strategy": "lines"
            }
            tables = page.find_tables(table_settings)
            for table in tables:
                table_bbox = table.bbox if hasattr(table, 'bbox') else None
                if table_bbox:
                    table_data_extracted = table.extract()
                    previous_table_extracted = previous_table.extract() if previous_table else None
                    if check_header_counts(previous_table, previous_table_extracted, table_data_extracted):
                        # Check if the table has more than one column
                        if is_current_table_at_top(page.height, table_bbox):
                            if previous_table is not None:
                                prev_header = extract_table_header(previous_table_extracted)
                                curr_header = extract_table_header(table_data_extracted)

                                if is_prev_table_at_right(page.width, previous_table.bbox)  and prev_header != curr_header:
                                    print(f"Headers  don't match on page {i+1}")
                                    merged_table_df = merge_tables_by_index(previous_table_extracted, table_data_extracted)
                                    merged_table_count += 1
                                    merged_table_df.to_csv(f'merged_table_{merged_table_count}.csv', index=False)
                                    print(f"Merged Table {merged_table_count} saved as merged_table_{merged_table_count}.csv")
                                if is_prev_table_at_right(page.width, previous_table.bbox)  and prev_header == curr_header:
                                    print(f"Headers match on page {i+1} merge regularly")
                                    merged_table_df = merge_tables_by_columns(previous_table_extracted, table_data_extracted)
                                    merged_table_count += 1
                                    merged_table_df.to_csv(f'merged_table_{merged_table_count}.csv', index=False)
                                    print(f"Merged Table {merged_table_count} saved as merged_table_{merged_table_count}.csv")
                                previous_table = None
                            else:
                                previous_table = table
                        else:
                            if previous_table is not None:
                                previous_table = table


def check_header_counts(previous_table, previous_table_extracted, table_data_extracted):
    return len(table_data_extracted[0]) > 1 or (previous_table and len(previous_table_extracted[0]) > 1)


pdf_path = 'test.pdf'
extract_tables_with_position_and_merge(pdf_path)