tableparser.py

Requirement =====================================================================================

        rowseps = {}
        for i in range(bottom - 1, top, -1):
            if self.block[i][left] == '+':
                rowseps[i] = [left]
            elif self.block[i][left] != '|':
                return None
        return rowseps

    def structure_from_cells(self):
        """
        From the data collected by `scan_cell()`, convert to the final data
        structure.
        """
        rowseps = self.rowseps.keys()   # list of row boundaries
        rowseps.sort()
        rowindex = {}
        for i in range(len(rowseps)):
            rowindex[rowseps[i]] = i    # row boundary -> row number mapping
        colseps = self.colseps.keys()   # list of column boundaries
        colseps.sort()
        colindex = {}
        for i in range(len(colseps)):
            colindex[colseps[i]] = i    # column boundary -> col number map
        colspecs = [(colseps[i] - colseps[i - 1] - 1)
                    for i in range(1, len(colseps))] # list of column widths
        # prepare an empty table with the correct number of rows & columns
        onerow = [None for i in range(len(colseps) - 1)]
        rows = [onerow[:] for i in range(len(rowseps) - 1)]
        # keep track of # of cells remaining; should reduce to zero
        remaining = (len(rowseps) - 1) * (len(colseps) - 1)
        for top, left, bottom, right, block in self.cells:
            rownum = rowindex[top]
            colnum = colindex[left]
            assert rows[rownum][colnum] is None, (
                  'Cell (row %s, column %s) already used.'
                  % (rownum + 1, colnum + 1))
            morerows = rowindex[bottom] - rownum - 1
            morecols = colindex[right] - colnum - 1
            remaining -= (morerows + 1) * (morecols + 1)
            # write the cell into the table
            rows[rownum][colnum] = (morerows, morecols, top + 1, block)
        assert remaining == 0, 'Unused cells remaining.'
        if self.head_body_sep:          # separate head rows from body rows
            numheadrows = rowindex[self.head_body_sep]
            headrows = rows[:numheadrows]
            bodyrows = rows[numheadrows:]
        else:
            headrows = []
            bodyrows = rows
        return (colspecs, headrows, bodyrows)


class SimpleTableParser(TableParser):

    """
    Parse a simple table using `parse()`.

    Here's an example of a simple table::

        =====  =====
        col 1  col 2
        =====  =====
        1      Second column of row 1.
        2      Second column of row 2.
               Second line of paragraph.
        3      - Second column of row 3.

               - Second item in bullet
                 list (row 3, column 2).
        4 is a span
        ------------
        5
        =====  =====

    Top and bottom borders use '=', column span underlines use '-', column
    separation is indicated with spaces.

    Passing the above table to the `parse()` method will result in the
    following data structure, whose interpretation is the same as for
    `GridTableParser`::

        ([5, 25],
         [[(0, 0, 1, ['col 1']),
           (0, 0, 1, ['col 2'])]],
         [[(0, 0, 3, ['1']),
           (0, 0, 3, ['Second column of row 1.'])],
          [(0, 0, 4, ['2']),
           (0, 0, 4, ['Second column of row 2.',
                      'Second line of paragraph.'])],
          [(0, 0, 6, ['3']),
           (0, 0, 6, ['- Second column of row 3.',
                      '',
                      '- Second item in bullet',
                      '  list (row 3, column 2).'])],
          [(0, 1, 10, ['4 is a span'])],
          [(0, 0, 12, ['5']),
           (0, 0, 12, [''])]])
    """

    head_body_separator_pat = re.compile('=[ =]*$')
    span_pat = re.compile('-[ -]*$')

    def setup(self, block):
        self.block = block[:]           # make a copy; it will be modified
        self.block.disconnect()         # don't propagate changes to parent
        # Convert top & bottom borders to column span underlines:
        self.block[0] = self.block[0].replace('=', '-')
        self.block[-1] = self.block[-1].replace('=', '-')
        self.head_body_sep = None
        self.columns = []
        self.border_end = None
        self.table = []
        self.done = [-1] * len(block[0])
        self.rowseps = {0: [0]}
        self.colseps = {0: [0]}

    def parse_table(self):
        """
        First determine the column boundaries from the top border, then
        process rows.  Each row may consist of multiple lines; accumulate
        lines until a row is complete.  Call `self.parse_row` to finish the
        job.
        """
        # Top border must fully describe all table columns.
        self.columns = self.parse_columns(self.block[0], 0)
        self.border_end = self.columns[-1][1]
        firststart, firstend = self.columns[0]
        offset = 1                      # skip top border
        start = 1
        text_found = None
        while offset < len(self.block):
            line = self.block[offset]
            if self.span_pat.match(line):
                # Column span underline or border; row is complete.
                self.parse_row(self.block[start:offset], start,
                               (line.rstrip(), offset))
                start = offset + 1
                text_found = None
            elif line[firststart:firstend].strip():
                # First column not blank, therefore it's a new row.
                if text_found and offset != start:
                    self.parse_row(self.block[start:offset], start)
                start = offset
                text_found = 1
            elif not text_found:
                start = offset + 1
            offset += 1

    def parse_columns(self, line, offset):
        """
        Given a column span underline, return a list of (begin, end) pairs.
        """
        cols = []
        end = 0
        while 1:
            begin = line.find('-', end)
            end = line.find(' ', begin)
            if begin < 0:
                break
            if end < 0:
                end = len(line)
            cols.append((begin, end))
        if self.columns:
            if cols[-1][1] != self.border_end:
                raise TableMarkupError('Column span incomplete at line '
                                       'offset %s.' % offset)
            # Allow for an unbounded rightmost column:
            cols[-1] = (cols[-1][0], self.columns[-1][1])
        return cols

    def init_row(self, colspec, offset):
        i = 0
        cells = []
        for start, end in colspec:
            morecols = 0
            try:
                assert start == self.columns[i][0]
                while end != self.columns[i][1]:
                    i += 1
                    morecols += 1
            except (AssertionError, IndexError):
                raise TableMarkupError('Column span alignment problem at '
                                       'line offset %s.' % (offset + 1))
            cells.append([0, morecols, offset, []])
            i += 1
        return cells

    def parse_row(self, lines, start, spanline=None):
        """
        Given the text `lines` of a row, parse it and append to `self.table`.

        The row is parsed according to the current column spec (either
        `spanline` if provided or `self.columns`).  For each column, extract
        text from each line, and check for text in column margins.  Finally,
        adjust for insigificant whitespace.
        """
        if not (lines or spanline):
            # No new row, just blank lines.
            return
        if spanline:
            columns = self.parse_columns(*spanline)
            span_offset = spanline[1]
        else:
            columns = self.columns[:]
            span_offset = start
        self.check_columns(lines, start, columns)
        row = self.init_row(columns, start)
        for i in range(len(columns)):
            start, end = columns[i]
            cellblock = lines.get_2D_block(0, start, len(lines), end)
            cellblock.disconnect()      # lines in cell can't sync with parent
            cellblock.replace(self.double_width_pad_char, '')
            row[i][3] = cellblock
        self.table.append(row)

    def check_columns(self, lines, first_line, columns):
        """
        Check for text in column margins and text overflow in the last column.
        Raise TableMarkupError if anything but whitespace is in column margins.
        Adjust the end value for the last column if there is text overflow.
        """
        # "Infinite" value for a dummy last column's beginning, used to
        # check for text overflow:
        columns.append((sys.maxint, None))
        lastcol = len(columns) - 2
        for i in range(len(columns) - 1):
            start, end = columns[i]
            nextstart = columns[i+1][0]
            offset = 0
            for line in lines:
                if i == lastcol and line[end:].strip():
                    text = line[start:].rstrip()
                    new_end = start + len(text)
                    columns[i] = (start, new_end)
                    main_start, main_end = self.columns[-1]
                    if new_end > main_end:
                        self.columns[-1] = (main_start, new_end)
                elif line[end:nextstart].strip():
                    raise TableMarkupError('Text in column margin at line '
                                           'offset %s.' % (first_line + offset))
                offset += 1
        columns.pop()

    def structure_from_cells(self):
        colspecs = [end - start for start, end in self.columns]
        first_body_row = 0
        if self.head_body_sep:
            for i in range(len(self.table)):
                if self.table[i][0][2] > self.head_body_sep:
                    first_body_row = i
                    break
        return (colspecs, self.table[:first_body_row],
                self.table[first_body_row:])


def update_dict_of_lists(master, newdata):
    """
    Extend the list values of `master` with those from `newdata`.

    Both parameters must be dictionaries containing list values.
    """
    for key, values in newdata.items():
        master.setdefault(key, []).extend(values)