Testing for EntitiesForbidden broken after 3.1 upgrade
Environment:
# Python 3.10
pip install pandas pytest defusedxml openpyxl
I have a test similar to this one in order to verify that an XLSX reader (declared here as reader
) is protected against malicious files:
from io import BytesIO
from zipfile import ZIP_DEFLATED
from zipfile import ZipFile
import defusedxml
from pandas import read_excel
from pytest import raises
def reader(filename):
"""
XLSX reader implementation that uses the openpyxl engine.
"""
return read_excel(filename, engine="openpyxl")
def test_reader_malicious_file():
"""
Malicious files should raise an exception.
Even though some up-to-date XML reader implementations are not vulnerable
to at least some of this attacks, we expect the exception to be raised by
defusedxml, the recommended package to safely read untrusted files:
https://docs.python.org/3.11/library/xml.html#the-defusedxml-package
"""
malicious_content = """<?xml version="1.0"?>
<!DOCTYPE lolz [
<!ENTITY lol "lol">
<!ELEMENT lolz (#PCDATA)>
<!ENTITY lol1 "&lol;&lol;&lol;&lol;&lol;&lol;&lol;&lol;&lol;&lol;">
<!ENTITY lol2 "&lol1;&lol1;&lol1;&lol1;&lol1;&lol1;&lol1;&lol1;&lol1;&lol1;">
<!ENTITY lol3 "&lol2;&lol2;&lol2;&lol2;&lol2;&lol2;&lol2;&lol2;&lol2;&lol2;">
<!ENTITY lol4 "&lol3;&lol3;&lol3;&lol3;&lol3;&lol3;&lol3;&lol3;&lol3;&lol3;">
<!ENTITY lol5 "&lol4;&lol4;&lol4;&lol4;&lol4;&lol4;&lol4;&lol4;&lol4;&lol4;">
<!ENTITY lol6 "&lol5;&lol5;&lol5;&lol5;&lol5;&lol5;&lol5;&lol5;&lol5;&lol5;">
<!ENTITY lol7 "&lol6;&lol6;&lol6;&lol6;&lol6;&lol6;&lol6;&lol6;&lol6;&lol6;">
<!ENTITY lol8 "&lol7;&lol7;&lol7;&lol7;&lol7;&lol7;&lol7;&lol7;&lol7;&lol7;">
<!ENTITY lol9 "&lol8;&lol8;&lol8;&lol8;&lol8;&lol8;&lol8;&lol8;&lol8;&lol8;">
]>
<lolz>&lol9;</lolz>"""
xlsx_buffer = BytesIO()
with ZipFile(xlsx_buffer, "a", ZIP_DEFLATED, False) as malicious_file:
malicious_file.writestr("[Content_Types].xml", malicious_content)
with raises(defusedxml.common.EntitiesForbidden):
reader(xlsx_buffer)
As you can see, the idea is to make sure that defusedxml
is actually being used by the reader
and it raises the appropriate exception for a malicious file.
This was working fine until 3.0.10
. However, with 3.1
it seems the exception is masked and, instead, a generic ValueError
is thrown instead:
Click to see complete output log
================================================================= test session starts =================================================================
platform linux -- Python 3.10.8, pytest-7.2.0, pluggy-1.0.0
rootdir: .../load, configfile: pyproject.toml
plugins: anyio-3.6.2, xdist-3.0.2, mock-3.10.0, flask-1.2.0, cov-4.0.0, profiling-1.7.0, clarity-1.0.1
collected 1 item
test_asdf.py F [100%]
====================================================================== FAILURES =======================================================================
_____________________________________________________________ test_reader_malicious_file ______________________________________________________________
self = <openpyxl.reader.excel.ExcelReader object at 0x7f894dbda650>
def read(self):
action = "read manifest"
try:
self.read_manifest()
.../envs/python/lib/python3.10/site-packages/openpyxl/reader/excel.py:287:
self = <openpyxl.reader.excel.ExcelReader object at 0x7f894dbda650>
def read_manifest(self):
src = self.archive.read(ARC_CONTENT_TYPES)
root = fromstring(src)
.../envs/python/lib/python3.10/site-packages/openpyxl/reader/excel.py:135:
text = b'\n \n \n \n \n ]>\n &lol9;'
forbid_dtd = False, forbid_entities = True, forbid_external = True
def fromstring(text, forbid_dtd=False, forbid_entities=True, forbid_external=True):
parser = DefusedXMLParser(
target=_TreeBuilder(),
forbid_dtd=forbid_dtd,
forbid_entities=forbid_entities,
forbid_external=forbid_external,
)
parser.feed(text)
.../envs/python/lib/python3.10/site-packages/defusedxml/common.py:126:
self = <defusedxml.ElementTree.DefusedXMLParser object at 0x7f894dbda620>
data = b'\n \n \n \n \n ]>\n &lol9;'
def feed(self, data):
"""Feed encoded data to parser."""
try:
self.parser.Parse(data, False)
.../envs/python/lib/python3.10/xml/etree/ElementTree.py:1713:
???
/croot/python-split_1669298683653/work/Modules/pyexpat.c:490:
self = <defusedxml.ElementTree.DefusedXMLParser object at 0x7f894dbda620>, name = 'lol', is_parameter_entity = 0, value = 'lol', base = None
sysid = None, pubid = None, notation_name = None
def defused_entity_decl(
self, name, is_parameter_entity, value, base, sysid, pubid, notation_name
):
raise EntitiesForbidden(name, value, base, sysid, pubid, notation_name)
E defusedxml.common.EntitiesForbidden: EntitiesForbidden(name='lol', system_id=None, public_id=None)
.../envs/python/lib/python3.10/site-packages/defusedxml/ElementTree.py:124: EntitiesForbidden
The above exception was the direct cause of the following exception:
def test_reader_malicious_file():
"""
Malicious files should raise an exception.
Even though some up-to-date XML reader implementations are not vulnerable
to at least some of this attacks, we expect the exception to be raised by
defusedxml, the recommended package to safely read untrusted files:
https://docs.python.org/3.11/library/xml.html#the-defusedxml-package
"""
malicious_content = """<?xml version="1.0"?>
<!DOCTYPE lolz [
<!ENTITY lol "lol">
<!ELEMENT lolz (#PCDATA)>
<!ENTITY lol1 "&lol;&lol;&lol;&lol;&lol;&lol;&lol;&lol;&lol;&lol;">
<!ENTITY lol2 "&lol1;&lol1;&lol1;&lol1;&lol1;&lol1;&lol1;&lol1;&lol1;&lol1;">
<!ENTITY lol3 "&lol2;&lol2;&lol2;&lol2;&lol2;&lol2;&lol2;&lol2;&lol2;&lol2;">
<!ENTITY lol4 "&lol3;&lol3;&lol3;&lol3;&lol3;&lol3;&lol3;&lol3;&lol3;&lol3;">
<!ENTITY lol5 "&lol4;&lol4;&lol4;&lol4;&lol4;&lol4;&lol4;&lol4;&lol4;&lol4;">
<!ENTITY lol6 "&lol5;&lol5;&lol5;&lol5;&lol5;&lol5;&lol5;&lol5;&lol5;&lol5;">
<!ENTITY lol7 "&lol6;&lol6;&lol6;&lol6;&lol6;&lol6;&lol6;&lol6;&lol6;&lol6;">
<!ENTITY lol8 "&lol7;&lol7;&lol7;&lol7;&lol7;&lol7;&lol7;&lol7;&lol7;&lol7;">
<!ENTITY lol9 "&lol8;&lol8;&lol8;&lol8;&lol8;&lol8;&lol8;&lol8;&lol8;&lol8;">
]>
<lolz>&lol9;</lolz>"""
xlsx_buffer = BytesIO()
with ZipFile(xlsx_buffer, "a", ZIP_DEFLATED, False) as malicious_file:
malicious_file.writestr("[Content_Types].xml", malicious_content)
with raises(defusedxml.common.EntitiesForbidden):
reader(xlsx_buffer)
test_asdf.py:43:
test_asdf.py:11: in reader
return read_excel(filename, engine="openpyxl")
.../envs/python/lib/python3.10/site-packages/pandas/util/_decorators.py:211: in wrapper
return func(*args, **kwargs)
.../envs/python/lib/python3.10/site-packages/pandas/util/_decorators.py:331: in wrapper
return func(*args, **kwargs)
.../envs/python/lib/python3.10/site-packages/pandas/io/excel/_base.py:482: in read_excel
io = ExcelFile(io, storage_options=storage_options, engine=engine)
.../envs/python/lib/python3.10/site-packages/pandas/io/excel/_base.py:1695: in init
self._reader = self._engines[engine](self._io, storage_options=storage_options)
.../envs/python/lib/python3.10/site-packages/pandas/io/excel/_openpyxl.py:557: in init
super().init(filepath_or_buffer, storage_options=storage_options)
.../envs/python/lib/python3.10/site-packages/pandas/io/excel/_base.py:545: in init
self.book = self.load_workbook(self.handles.handle)
.../envs/python/lib/python3.10/site-packages/pandas/io/excel/_openpyxl.py:568: in load_workbook
return load_workbook(
.../envs/python/lib/python3.10/site-packages/openpyxl/reader/excel.py:346: in load_workbook
reader.read()
self = <openpyxl.reader.excel.ExcelReader object at 0x7f894dbda650>
def read(self):
action = "read manifest"
try:
self.read_manifest()
action = "read strings"
self.read_strings()
action = "read workbook"
self.read_workbook()
action = "read properties"
self.read_properties()
action = "read custom properties"
self.read_custom()
action = "read theme"
self.read_theme()
action = "read stylesheet"
apply_stylesheet(self.archive, self.wb)
action = "read worksheets"
self.read_worksheets()
action = "assign names"
self.parser.assign_names()
if not self.read_only:
self.archive.close()
except ValueError as e:
raise ValueError(
f"Unable to read workbook: could not {action} from {self.archive.filename}.\n"
"This is most probably because the workbook source files contain some invalid XML.\n"
"Please see the exception for more details."
) from e
E ValueError: Unable to read workbook: could not read manifest from None.
E This is most probably because the workbook source files contain some invalid XML.
E Please see the exception for more details.
.../envs/python/lib/python3.10/site-packages/openpyxl/reader/excel.py:307: ValueError
=============================================================== short test summary info ===============================================================
FAILED test_asdf.py::test_reader_malicious_file - ValueError: Unable to read workbook: could not read manifest from None.
================================================================== 1 failed in 0.26s ==================================================================
Is this an intentional change? If so, how should I implement the test in order to check for the actual EntitiesForbidden
exception?
Edited by Miguel Sánchez de León Peque